@agentsean/crawler 1.0.0

package/dist/adaptive.js

@@ -0,0 +1,43 @@
+import { jsDependencyScore } from "./render.js";
+const DETECTION_RATIO = 0.1;
+const FINGERPRINT_N = 50;
+export function decideRenderPolicy(pages, page, index) {
+    if (shouldAlwaysRender(page))
+        return "always";
+    if (index < FINGERPRINT_N)
+        return "sample";
+    const same = pages.filter((p) => p.templateKey === page.templateKey);
+    if (same.length < 3)
+        return "sample";
+    const scores = same.map((p) => p.jsDependencyScore).toSorted((a, b) => a - b);
+    const median = scores[Math.floor(scores.length / 2)] ?? 0;
+    const p95 = scores[Math.floor(scores.length * 0.95)] ?? median;
+    if (median > 0.35)
+        return "always";
+    if (p95 < 0.05)
+        return "never";
+    return Math.random() < DETECTION_RATIO ? "sample" : "never";
+}
+export function shouldAlwaysRender(page) {
+    try {
+        const u = new URL(page.url);
+        if (u.pathname === "/" || u.pathname === "")
+            return true;
+    }
+    catch {
+        /* ignore */
+    }
+    if (!page.extract)
+        return false;
+    if (page.extract.spaRootEmpty)
+        return true;
+    if (page.extract.hasNoscriptJsWarning)
+        return true;
+    if (page.extract.mainWordCount < 200 && page.extract.spaRootEmpty)
+        return true;
+    return false;
+}
+export function scoreFromHtml(html, extract) {
+    return jsDependencyScore(html, extract);
+}
+//# sourceMappingURL=adaptive.js.map

package/dist/adaptive.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive.js","sourceRoot":"","sources":["../src/adaptive.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAKhD,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,UAAU,kBAAkB,CAChC,KAAuE,EACvE,IAAgF,EAChF,KAAa;IAEb,IAAI,kBAAkB,CAAC,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9C,IAAI,KAAK,GAAG,aAAa;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,WAAW,CAAC,CAAC;IACrE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC;IAC/D,IAAI,MAAM,GAAG,IAAI;QAAE,OAAO,QAAQ,CAAC;IACnC,IAAI,GAAG,GAAG,IAAI;QAAE,OAAO,OAAO,CAAC;IAC/B,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAGlC;IACC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,CAAC,QAAQ,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,YAAY;IACd,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IAC/E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,OAAoB;IAC9D,OAAO,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC"}

package/dist/crawl.d.ts

@@ -0,0 +1,4 @@
+import type { CrawlOptions, CrawlResult } from "./types.js";
+export declare function ramConcurrency(): number;
+export declare function crawlSite(options: CrawlOptions): Promise<CrawlResult>;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAGV,YAAY,EACZ,WAAW,EAGZ,MAAM,YAAY,CAAC;AAEpB,wBAAgB,cAAc,IAAI,MAAM,CAKvC;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAuK3E"}

package/dist/crawl.js

@@ -0,0 +1,348 @@
+import os from "node:os";
+import { createCrawlerAgent, fetchUrl, ROBOTS_MAX_REDIRECTS } from "./http.js";
+import { extractPage, parseLinkHeaderCanonicals } from "./extract.js";
+import { parseSitemapXml } from "./sitemap.js";
+import { isAllowedByRobots, robotsFromFetch } from "./robots.js";
+import { contentHash } from "./hash.js";
+import { simhashHex } from "./simhash.js";
+import { originOf, templateKey, isInternalUrl, normalizeUrl } from "./url.js";
+import { decideRenderPolicy, scoreFromHtml } from "./adaptive.js";
+import { renderPage } from "./render.js";
+import { probeOrigin } from "./probe.js";
+import { SEAN_UA } from "./ua.js";
+export function ramConcurrency() {
+    const freeMb = os.freemem() / (1024 * 1024);
+    const cores = Math.max(1, os.cpus().length);
+    const fromRam = Math.floor((freeMb - 400) / 50);
+    return Math.max(2, Math.min(fromRam || cores, cores * 2, 16));
+}
+export async function crawlSite(options) {
+    const startUrl = normalizeUrl(options.startUrl);
+    const origin = originOf(startUrl);
+    const maxPages = options.maxPages ?? 5000;
+    const concurrency = options.concurrency ?? ramConcurrency();
+    const rps = options.rps ?? 4;
+    const render = options.render ?? true;
+    const userAgent = options.userAgent ?? SEAN_UA;
+    const startedAt = new Date().toISOString();
+    const agent = createCrawlerAgent(2);
+    const robots = await fetchRobots(origin, agent, userAgent);
+    const sitemaps = await fetchSitemaps(origin, robots, agent, userAgent);
+    const seen = new Set(options.resume?.seen ?? []);
+    const queue = options.resume?.queue
+        ? options.resume.queue.map((q) => ({ url: q.url, depth: q.depth }))
+        : [];
+    const enqueue = (url, depth) => {
+        let abs;
+        try {
+            abs = normalizeUrl(url);
+        }
+        catch {
+            return;
+        }
+        if (!isInternalUrl(abs, origin) && !options.followExternal)
+            return;
+        if (seen.has(abs))
+            return;
+        if (!isAllowedByRobots(robots, abs, userAgent)) {
+            seen.add(abs);
+            blocked.push(abs);
+            return;
+        }
+        seen.add(abs);
+        queue.push({ url: abs, depth });
+    };
+    const blocked = [];
+    if (!options.resume) {
+        enqueue(startUrl, 0);
+        for (const sm of sitemaps) {
+            for (const u of sm.urls)
+                enqueue(u.loc, 1);
+        }
+    }
+    const sitemapLocs = new Set(sitemaps.flatMap((s) => s.urls.map((u) => normalizeUrlSafe(u.loc))));
+    const pages = [];
+    const inlinks = new Map();
+    const followInlinks = new Map();
+    const minInterval = 1000 / Math.max(0.1, rps);
+    let lastFetch = 0;
+    let nextIndex = 0;
+    let active = 0;
+    const snapshot = () => ({
+        seen: [...seen],
+        queue: queue.map((q) => ({ url: q.url, depth: q.depth })),
+        pagesSeen: pages.length,
+    });
+    const emitCheckpoint = () => {
+        const every = options.checkpointEvery ?? 10;
+        if (pages.length % every !== 0)
+            return;
+        options.onCheckpoint?.(snapshot());
+    };
+    let aborted = false;
+    const workers = Array.from({ length: Math.min(concurrency, maxPages) }, async () => {
+        while (pages.length < maxPages) {
+            if (options.signal?.aborted) {
+                aborted = true;
+                return;
+            }
+            const item = queue.shift();
+            if (!item) {
+                if (active === 0)
+                    return;
+                await sleep(20);
+                continue;
+            }
+            active++;
+            try {
+                const wait = lastFetch + minInterval - Date.now();
+                if (wait > 0)
+                    await sleep(wait);
+                lastFetch = Date.now();
+                const page = await crawlOne({
+                    url: item.url,
+                    depth: item.depth,
+                    origin,
+                    agent,
+                    userAgent,
+                    timeoutMs: options.timeoutMs ?? 20_000,
+                    robots,
+                    sitemapLocs,
+                    render,
+                    index: nextIndex++,
+                    prior: pages,
+                    check304: options.check304 ?? true,
+                });
+                pages.push(page);
+                emitCheckpoint();
+                if (page.extract) {
+                    for (const link of page.extract.links) {
+                        if (!link.isInternal)
+                            continue;
+                        inlinks.set(link.absUrl, (inlinks.get(link.absUrl) ?? 0) + 1);
+                        if (!link.isNofollow) {
+                            followInlinks.set(link.absUrl, (followInlinks.get(link.absUrl) ?? 0) + 1);
+                        }
+                        if (pages.length + queue.length < maxPages * 2) {
+                            enqueue(link.absUrl, item.depth + 1);
+                        }
+                    }
+                }
+            }
+            finally {
+                active--;
+            }
+        }
+    });
+    await Promise.all(workers);
+    for (const p of pages) {
+        p.inlinkCount = inlinks.get(p.url) ?? 0;
+        p.followInlinkCount = followInlinks.get(p.url) ?? 0;
+    }
+    let originProbe;
+    try {
+        originProbe = await probeOrigin(startUrl, agent);
+    }
+    catch {
+        originProbe = {
+            https: startUrl.startsWith("https:"),
+            httpRedirectsToHttps: null,
+            wwwSplit: null,
+            wwwPreferred: null,
+            trailingSlashSplit: null,
+            randomSoft404: false,
+            randomSoft404Url: null,
+            randomSoft404Hash: null,
+            certValidTo: null,
+            certDaysRemaining: null,
+            certError: null,
+            alpn: null,
+            hsts: null,
+        };
+    }
+    await agent.close();
+    const checkpoint = snapshot();
+    options.onCheckpoint?.(checkpoint);
+    return {
+        origin,
+        startUrl,
+        startedAt,
+        finishedAt: new Date().toISOString(),
+        pages,
+        robots,
+        sitemaps,
+        originProbe,
+        pagesSeen: pages.length,
+        pagesChanged: pages.filter((p) => !p.notModified).length,
+        maxPages,
+        truncated: queue.length > 0 || seen.size > pages.length,
+        aborted,
+        checkpoint,
+    };
+}
+async function crawlOne(opts) {
+    const fetched = await fetchUrl(opts.url, {
+        agent: opts.agent,
+        userAgent: opts.userAgent,
+        timeoutMs: opts.timeoutMs,
+    });
+    const isHtml = /html|xml|xhtml/i.test(fetched.contentType) || looksLikeHtml(fetched.decoded);
+    const html = isHtml ? fetched.decoded.toString("utf8") : null;
+    const headerCanonical = parseLinkHeaderCanonicals(fetched.headers.link);
+    const xRobots = fetched.headers["x-robots-tag"] ?? null;
+    const extract = html
+        ? extractPage(html, fetched.finalUrl, headerCanonical, xRobots)
+        : null;
+    const hash = fetched.decoded.length ? contentHash(fetched.decoded) : null;
+    const sh = extract ? simhashHex(extract.mainText || extract.allText) : null;
+    const jsScore = html && extract ? scoreFromHtml(html, extract) : 0;
+    const tmpl = templateKey(opts.url);
+    let supports304 = null;
+    if (opts.check304 &&
+        fetched.statusCode === 200 &&
+        (fetched.etag || fetched.lastModified)) {
+        const again = await fetchUrl(opts.url, {
+            agent: opts.agent,
+            userAgent: opts.userAgent,
+            timeoutMs: opts.timeoutMs,
+            etag: fetched.etag ?? undefined,
+            lastModified: fetched.lastModified ?? undefined,
+            maxRedirects: 0,
+        });
+        supports304 = again.statusCode === 304;
+    }
+    const draft = {
+        url: opts.url,
+        finalUrl: fetched.finalUrl,
+        statusCode: fetched.statusCode,
+        error: fetched.error,
+        contentType: fetched.contentType,
+        headers: fetched.headers,
+        ttfbMs: fetched.ttfbMs,
+        totalMs: fetched.totalMs,
+        wireBytes: fetched.wireBytes,
+        decodedBytes: fetched.decodedBytes,
+        httpVersion: null,
+        redirectChain: fetched.redirectChain,
+        redirectLoop: fetched.redirectLoop,
+        exceedsGoogleRedirectLimit: fetched.exceedsGoogleRedirectLimit,
+        html,
+        renderedHtml: null,
+        extract,
+        renderedExtract: null,
+        contentHash: hash,
+        simhash: sh,
+        etag: fetched.etag,
+        lastModified: fetched.lastModified,
+        notModified: fetched.notModified,
+        supports304,
+        inSitemap: opts.sitemapLocs.has(opts.url) || opts.sitemapLocs.has(fetched.finalUrl),
+        robotsAllowed: isAllowedByRobots(opts.robots, opts.url, opts.userAgent),
+        depth: opts.depth,
+        inlinkCount: 0,
+        outlinkCount: extract?.links.filter((l) => l.isInternal).length ?? 0,
+        followInlinkCount: 0,
+        isInternal: isInternalUrl(opts.url, opts.origin),
+        fetchedAt: new Date().toISOString(),
+        renderPolicy: "skipped",
+        rendered: false,
+        jsDependencyScore: jsScore,
+        templateKey: tmpl,
+    };
+    if (opts.render && html && extract && fetched.statusCode === 200) {
+        const policy = decideRenderPolicy(opts.prior, draft, opts.index);
+        draft.renderPolicy = policy === "never" ? "never" : policy;
+        if (policy === "always" || policy === "sample") {
+            const rendered = await renderPage(opts.url, html);
+            if (rendered && !rendered.errors.includes("playwright_unavailable")) {
+                draft.renderedHtml = rendered.html;
+                draft.renderedExtract = rendered.extract;
+                draft.rendered = true;
+                if (rendered.timedOut)
+                    draft.error = draft.error ?? "render_timeout";
+            }
+            else {
+                draft.renderPolicy = "skipped";
+            }
+        }
+    }
+    return draft;
+}
+async function fetchRobots(origin, agent, userAgent) {
+    const url = `${origin}/robots.txt`;
+    const res = await fetchUrl(url, {
+        agent,
+        userAgent,
+        timeoutMs: 10_000,
+        maxRedirects: ROBOTS_MAX_REDIRECTS,
+        accept: "text/plain,*/*;q=0.1",
+    });
+    return robotsFromFetch({
+        statusCode: res.statusCode,
+        raw: res.decoded.toString("utf8"),
+        contentType: res.contentType,
+        bytes: res.decodedBytes,
+        redirectHops: Math.max(0, res.redirectChain.length - 1),
+        error: res.error,
+    });
+}
+async function fetchSitemaps(origin, robots, agent, userAgent) {
+    const candidates = [
+        ...robots.sitemaps,
+        `${origin}/sitemap.xml`,
+        `${origin}/sitemap_index.xml`,
+    ];
+    const seen = new Set();
+    const out = [];
+    for (const loc of candidates) {
+        let url;
+        try {
+            url = new URL(loc, origin).href;
+        }
+        catch {
+            continue;
+        }
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        const res = await fetchUrl(url, {
+            agent,
+            userAgent,
+            timeoutMs: 15_000,
+            accept: "application/xml,text/xml,*/*;q=0.1",
+        });
+        if (res.statusCode === 404 || res.statusCode === null) {
+            if (!robots.sitemaps.includes(loc) && !robots.sitemaps.includes(url))
+                continue;
+        }
+        const parsed = await parseSitemapXml(res.decoded, url, url.endsWith(".gz") || (res.headers["content-type"] ?? "").includes("gzip"));
+        const sm = {
+            ...parsed,
+            statusCode: res.statusCode,
+            contentType: res.contentType,
+            error: parsed.error ?? res.error,
+        };
+        out.push(sm);
+        for (const child of sm.childSitemaps) {
+            if (!seen.has(child) && out.length < 50)
+                candidates.push(child);
+        }
+    }
+    return out;
+}
+function looksLikeHtml(buf) {
+    const head = buf.slice(0, 256).toString("utf8").toLowerCase();
+    return (head.includes("<html") || head.includes("<!doctype html") || head.includes("<head"));
+}
+function normalizeUrlSafe(url) {
+    try {
+        return normalizeUrl(url);
+    }
+    catch {
+        return url;
+    }
+}
+function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+//# sourceMappingURL=crawl.js.map

package/dist/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAC/E,OAAO,EAAE,WAAW,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC9E,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAUlC,MAAM,UAAU,cAAc;IAC5B,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAqB;IACnD,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,cAAc,EAAE,CAAC;IAC5D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACtC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAEpC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAEvE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,KAAK,GAAqC,OAAO,CAAC,MAAM,EAAE,KAAK;QACnE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnE,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,EAAE;QAC7C,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc;YAAE,OAAO;QACnE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAC1B,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC;IAEF,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACrB,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,CACzB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CACpE,CAAC;IACF,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAG,GAAoB,EAAE,CAAC,CAAC;QACvC,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QACf,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACzD,SAAS,EAAE,KAAK,CAAC,MAAM;KACxB,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,GAAG,EAAE;QAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,KAAK,CAAC;YAAE,OAAO;QACvC,OAAO,CAAC,YAAY,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrC,CAAC,CAAC;IACF,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACjF,OAAO,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC5B,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,IAAI,MAAM,KAAK,CAAC;oBAAE,OAAO;gBACzB,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YACD,MAAM,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,IAAI,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC;oBAC1B,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM;oBACN,KAAK;oBACL,SAAS;oBACT,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;oBACtC,MAAM;oBACN,WAAW;oBACX,MAAM;oBACN,KAAK,EAAE,SAAS,EAAE;oBAClB,KAAK,EAAE,KAAK;oBACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;iBACnC,CAAC,CAAC;gBACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,cAAc,EAAE,CAAC;gBACjB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;wBACtC,IAAI,CAAC,IAAI,CAAC,UAAU;4BAAE,SAAS;wBAC/B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBAC9D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;4BACrB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBAC5E,CAAC;wBACD,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,GAAG,CAAC,EAAE,CAAC;4BAC/C,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;wBACvC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC,CAAC,iBAAiB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,WAAW,GAAG;YACZ,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC;YACpC,oBAAoB,EAAE,IAAI;YAC1B,QAAQ,EAAE,IAAI;YACd,YAAY,EAAE,IAAI;YAClB,kBAAkB,EAAE,IAAI;YACxB,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,WAAW,EAAE,IAAI;YACjB,iBAAiB,EAAE,IAAI;YACvB,SAAS,EAAE,IAAI;YACf,IAAI,EAAE,IAAI;YACV,IAAI,EAAE,IAAI;SACX,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAEpB,MAAM,UAAU,GAAG,QAAQ,EAAE,CAAC;IAC9B,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;IAEnC,OAAO;QACL,MAAM;QACN,QAAQ;QACR,SAAS;QACT,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,KAAK;QACL,MAAM;QACN,QAAQ;QACR,WAAW;QACX,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM;QACxD,QAAQ;QACR,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM;QACvD,OAAO;QACP,UAAU;KACX,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAavB;IACC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE;QACvC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,MAAM,MAAM,GACV,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,MAAM,eAAe,GAAG,yBAAyB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;IACxD,MAAM,OAAO,GAAG,IAAI;QAClB,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,eAAe,EAAE,OAAO,CAAC;QAC/D,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,MAAM,OAAO,GAAG,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEnC,IAAI,WAAW,GAAmB,IAAI,CAAC;IACvC,IACE,IAAI,CAAC,QAAQ;QACb,OAAO,CAAC,UAAU,KAAK,GAAG;QAC1B,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,YAAY,CAAC,EACtC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS;YAC/B,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS;YAC/C,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;QACH,WAAW,GAAG,KAAK,CAAC,UAAU,KAAK,GAAG,CAAC;IACzC,CAAC;IAED,MAAM,KAAK,GAAgB;QACzB,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,WAAW,EAAE,IAAI;QACjB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;QAC9D,IAAI;QACJ,YAAY,EAAE,IAAI;QAClB,OAAO;QACP,eAAe,EAAE,IAAI;QACrB,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,WAAW;QACX,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;QACnF,aAAa,EAAE,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC;QACvE,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,IAAI,CAAC;QACpE,iBAAiB,EAAE,CAAC;QACpB,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;QAChD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,YAAY,EAAE,SAAS;QACvB,QAAQ,EAAE,KAAK;QACf,iBAAiB,EAAE,OAAO;QAC1B,WAAW,EAAE,IAAI;KAClB,CAAC;IAEF,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,OAAO,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QACjE,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,KAAK,CAAC,YAAY,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3D,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAClD,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACnC,KAAK,CAAC,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC;gBACzC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACtB,IAAI,QAAQ,CAAC,QAAQ;oBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,gBAAgB,CAAC;YACvE,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAc,EACd,KAAY,EACZ,SAAiB;IAEjB,MAAM,GAAG,GAAG,GAAG,MAAM,aAAa,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE;QAC9B,KAAK;QACL,SAAS;QACT,SAAS,EAAE,MAAM;QACjB,YAAY,EAAE,oBAAoB;QAClC,MAAM,EAAE,sBAAsB;KAC/B,CAAC,CAAC;IACH,OAAO,eAAe,CAAC;QACrB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjC,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,KAAK,EAAE,GAAG,CAAC,YAAY;QACvB,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;QACvD,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAc,EACd,MAAqB,EACrB,KAAY,EACZ,SAAiB;IAEjB,MAAM,UAAU,GAAG;QACjB,GAAG,MAAM,CAAC,QAAQ;QAClB,GAAG,MAAM,cAAc;QACvB,GAAG,MAAM,oBAAoB;KAC9B,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE;YAC9B,KAAK;YACL,SAAS;YACT,SAAS,EAAE,MAAM;YACjB,MAAM,EAAE,oCAAoC;SAC7C,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;YACtD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,SAAS;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,GAAG,CAAC,OAAO,EACX,GAAG,EACH,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC5E,CAAC;QACF,MAAM,EAAE,GAAkB;YACxB,GAAG,MAAM;YACT,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG,CAAC,KAAK;SACjC,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACb,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,aAAa,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,EAAE;gBAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9D,OAAO,CACL,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CACpF,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/decode.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Decode Content-Encoding. undici.request() does not decompress; we must.
+ * Apply chained encodings right-to-left.
+ */
+export declare function decodeBody(buf: Buffer, contentEncoding: string | undefined): Promise<Buffer>;
+export declare function acceptEncodingHeader(): string;
+//# sourceMappingURL=decode.d.ts.map

package/dist/decode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decode.d.ts","sourceRoot":"","sources":["../src/decode.ts"],"names":[],"mappings":"AAYA;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,MAAM,GAAG,SAAS,GAClC,OAAO,CAAC,MAAM,CAAC,CA2BjB;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAI7C"}

package/dist/decode.js

@@ -0,0 +1,53 @@
+import { promisify } from "node:util";
+import zlib from "node:zlib";
+const gunzip = promisify(zlib.gunzip);
+const inflate = promisify(zlib.inflate);
+const inflateRaw = promisify(zlib.inflateRaw);
+const brotli = promisify(zlib.brotliDecompress);
+const zstd = "zstdDecompress" in zlib && typeof zlib.zstdDecompress === "function"
+    ? promisify(zlib.zstdDecompress)
+    : null;
+/**
+ * Decode Content-Encoding. undici.request() does not decompress; we must.
+ * Apply chained encodings right-to-left.
+ */
+export async function decodeBody(buf, contentEncoding) {
+    if (!contentEncoding)
+        return buf;
+    const encodings = contentEncoding
+        .split(",")
+        .map((s) => s.trim().toLowerCase())
+        .filter((s) => s && s !== "identity");
+    let out = buf;
+    for (let i = encodings.length - 1; i >= 0; i--) {
+        const enc = encodings[i];
+        if (enc === "gzip" || enc === "x-gzip") {
+            out = Buffer.from(await gunzip(out));
+        }
+        else if (enc === "deflate") {
+            try {
+                out = Buffer.from(await inflate(out));
+            }
+            catch {
+                out = Buffer.from(await inflateRaw(out));
+            }
+        }
+        else if (enc === "br") {
+            out = Buffer.from(await brotli(out));
+        }
+        else if (enc === "zstd" || enc === "zst") {
+            if (!zstd) {
+                throw new Error("zstd Content-Encoding requires Node >= 22.15");
+            }
+            out = Buffer.from(await zstd(out));
+        }
+    }
+    return out;
+}
+export function acceptEncodingHeader() {
+    const parts = ["gzip", "deflate", "br"];
+    if (zstd)
+        parts.push("zstd");
+    return parts.join(", ");
+}
+//# sourceMappingURL=decode.js.map

package/dist/decode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decode.js","sourceRoot":"","sources":["../src/decode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACtC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACxC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;AAChD,MAAM,IAAI,GACR,gBAAgB,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,cAAc,KAAK,UAAU;IACnE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;IAChC,CAAC,CAAC,IAAI,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,GAAW,EACX,eAAmC;IAEnC,IAAI,CAAC,eAAe;QAAE,OAAO,GAAG,CAAC;IACjC,MAAM,SAAS,GAAG,eAAe;SAC9B,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,CAAC;IACxC,IAAI,GAAG,GAAW,GAAG,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;YACvC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;aAAM,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACxB,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;aAAM,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YACD,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IACxC,IAAI,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/extract.d.ts

@@ -0,0 +1,4 @@
+import type { PageExtract } from "./types.js";
+export declare function extractPage(html: string, pageUrl: string, headerCanonical?: string[], xRobots?: string | null): PageExtract;
+export declare function parseLinkHeaderCanonicals(header: string | undefined): string[];
+//# sourceMappingURL=extract.d.ts.map

package/dist/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAKV,WAAW,EACZ,MAAM,YAAY,CAAC;AAKpB,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,eAAe,GAAE,MAAM,EAAO,EAC9B,OAAO,GAAE,MAAM,GAAG,IAAW,GAC5B,WAAW,CA4Mb;AAiED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAQ9E"}