@agentsean/crawler 1.0.0

package/dist/robots.js

@@ -0,0 +1,187 @@
+/**
+ * Vendored robots.txt matcher.
+ *
+ * `robots-parser` 3.0.1 last published 2023-02-21 (stale). We do not depend
+ * on it. This implements RFC 9309 matching (most-octets wins, allow wins ties)
+ * with Google-style group selection (single most-specific user-agent group,
+ * no merge across product tokens).
+ *
+ * Status handling (both modes are RFC 9309 compliant):
+ * - 4xx except 429 → allow-all (RFC §2.3.1.3)
+ * - 5xx / 429 / network → polite-crawl: disallow-all (RFC §2.3.1.4 MUST)
+ */
+import { contentHash } from "./hash.js";
+import { pathOf } from "./url.js";
+const KNOWN = new Set(["user-agent", "allow", "disallow", "sitemap", "crawl-delay"]);
+export function parseRobotsTxt(raw) {
+    const lines = raw.replace(/^\uFEFF/, "").split(/\r?\n/);
+    const groups = [];
+    const sitemaps = [];
+    const unknownDirectives = [];
+    let hasNoindexDirective = false;
+    let current = null;
+    let pendingAgents = [];
+    const flushAgents = () => {
+        if (pendingAgents.length === 0)
+            return;
+        current = { agents: pendingAgents, rules: [], crawlDelay: null };
+        groups.push(current);
+        pendingAgents = [];
+    };
+    for (const rawLine of lines) {
+        const line = rawLine.replace(/#.*$/, "").trim();
+        if (!line)
+            continue;
+        const colon = line.indexOf(":");
+        if (colon < 1) {
+            unknownDirectives.push(rawLine.trim());
+            continue;
+        }
+        const field = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (field === "user-agent") {
+            if (current && current.rules.length + (current.crawlDelay ? 1 : 0) > 0) {
+                pendingAgents = [value.toLowerCase()];
+                current = null;
+            }
+            else if (pendingAgents.length > 0 && current === null) {
+                pendingAgents.push(value.toLowerCase());
+            }
+            else if (current && current.rules.length === 0 && !current.crawlDelay) {
+                current.agents.push(value.toLowerCase());
+            }
+            else {
+                pendingAgents = [value.toLowerCase()];
+                current = null;
+            }
+            continue;
+        }
+        if (field === "sitemap") {
+            sitemaps.push(value);
+            continue;
+        }
+        if (field === "noindex") {
+            hasNoindexDirective = true;
+            continue;
+        }
+        if (!KNOWN.has(field)) {
+            unknownDirectives.push(`${field}: ${value}`);
+            continue;
+        }
+        flushAgents();
+        if (!current) {
+            current = { agents: ["*"], rules: [], crawlDelay: null };
+            groups.push(current);
+        }
+        if (field === "allow") {
+            current.rules.push({ type: "allow", pattern: value });
+        }
+        else if (field === "disallow") {
+            current.rules.push({ type: "disallow", pattern: value });
+        }
+        else if (field === "crawl-delay") {
+            const n = Number(value);
+            current.crawlDelay = Number.isFinite(n) ? n : current.crawlDelay;
+        }
+    }
+    const hasCrawlDelay = groups.some((g) => g.crawlDelay !== null);
+    const crawlDelay = groups.find((g) => g.crawlDelay !== null)?.crawlDelay ?? null;
+    const disallowAll = groups.some((g) => (g.agents.includes("*") || g.agents.includes("googlebot")) &&
+        g.rules.some((r) => r.type === "disallow" && (r.pattern === "/" || r.pattern === "/*")));
+    return {
+        raw,
+        groups,
+        sitemaps,
+        unknownDirectives,
+        hasNoindexDirective,
+        hasCrawlDelay,
+        crawlDelay,
+        disallowAll,
+        hash: contentHash(raw),
+    };
+}
+export function robotsFromFetch(opts) {
+    const serverError = opts.error !== null ||
+        opts.statusCode === 429 ||
+        (opts.statusCode !== null && opts.statusCode >= 500);
+    if (serverError) {
+        const parsed = parseRobotsTxt(opts.raw || "");
+        return {
+            ...parsed,
+            ...opts,
+            mode: "disallow-all",
+            groups: parsed.groups,
+        };
+    }
+    if (opts.statusCode !== null && opts.statusCode >= 400 && opts.statusCode < 500) {
+        const parsed = parseRobotsTxt("");
+        return { ...parsed, ...opts, raw: opts.raw, mode: "allow-all" };
+    }
+    const parsed = parseRobotsTxt(opts.raw);
+    return { ...parsed, ...opts, mode: "parsed" };
+}
+export function isAllowedByRobots(robots, url, userAgent = "seanbot") {
+    if (robots.mode === "allow-all")
+        return true;
+    if (robots.mode === "disallow-all")
+        return false;
+    const path = pathOf(url);
+    const group = selectGroup(robots.groups, userAgent);
+    if (!group)
+        return true;
+    return mostSpecificAllows(group, path);
+}
+function selectGroup(groups, ua) {
+    const uaLower = ua.toLowerCase();
+    const matching = groups.filter((g) => g.agents.some((a) => a === "*" || uaLower.includes(a)));
+    const specific = matching.filter((g) => g.agents.some((a) => a !== "*"));
+    if (specific.length > 0) {
+        specific.sort((a, b) => longestAgent(b) - longestAgent(a));
+        return specific[0];
+    }
+    return matching.find((g) => g.agents.includes("*"));
+}
+function longestAgent(g) {
+    return Math.max(0, ...g.agents.map((a) => (a === "*" ? 0 : a.length)));
+}
+function mostSpecificAllows(group, path) {
+    let bestOctets = -1;
+    let allowed = true;
+    for (const rule of group.rules) {
+        if (!rule.pattern)
+            continue;
+        if (!matchRule(rule.pattern, path))
+            continue;
+        const octets = Buffer.byteLength(rule.pattern);
+        if (octets > bestOctets) {
+            bestOctets = octets;
+            allowed = rule.type === "allow";
+        }
+        else if (octets === bestOctets && rule.type === "allow") {
+            allowed = true;
+        }
+    }
+    return allowed;
+}
+export function matchRule(pattern, path) {
+    let end = false;
+    let p = pattern;
+    if (p.endsWith("$")) {
+        end = true;
+        p = p.slice(0, -1);
+    }
+    let re = "^";
+    for (const ch of p) {
+        if (ch === "*")
+            re += ".*";
+        else
+            re += escapeRe(ch);
+    }
+    if (end)
+        re += "$";
+    return new RegExp(re).test(path);
+}
+function escapeRe(ch) {
+    return ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+//# sourceMappingURL=robots.js.map

package/dist/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAGlC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;AAErF,MAAM,UAAU,cAAc,CAC5B,GAAW;IAKX,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,IAAI,aAAa,GAAa,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAG,GAAG,EAAE;QACvB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,OAAO,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,aAAa,GAAG,EAAE,CAAC;IACrB,CAAC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACvC,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE3C,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvE,aAAa,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtC,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,aAAa,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtC,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,mBAAmB,GAAG,IAAI,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,iBAAiB,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;YAC7C,SAAS;QACX,CAAC;QAED,WAAW,EAAE,CAAC;QACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;aAAM,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACxB,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QACnE,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,EAAE,UAAU,IAAI,IAAI,CAAC;IACjF,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAC7B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC1D,CAAC,CAAC,KAAK,CAAC,IAAI,CACV,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAC1E,CACJ,CAAC;IAEF,OAAO;QACL,GAAG;QACH,MAAM;QACN,QAAQ;QACR,iBAAiB;QACjB,mBAAmB;QACnB,aAAa;QACb,UAAU;QACV,WAAW;QACX,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC;KACvB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAO/B;IACC,MAAM,WAAW,GACf,IAAI,CAAC,KAAK,KAAK,IAAI;QACnB,IAAI,CAAC,UAAU,KAAK,GAAG;QACvB,CAAC,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC;IACvD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO;YACL,GAAG,MAAM;YACT,GAAG,IAAI;YACP,IAAI,EAAE,cAAc;YACpB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,GAAG,IAAI,IAAI,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QAChF,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,MAAqB,EACrB,GAAW,EACX,SAAS,GAAG,SAAS;IAErB,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,MAAM,CAAC,IAAI,KAAK,cAAc;QAAE,OAAO,KAAK,CAAC;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAAC,MAAqB,EAAE,EAAU;IACpD,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACnC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACvD,CAAC;IACF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,YAAY,CAAC,CAAc;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAkB,EAAE,IAAY;IAC1D,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,SAAS;QAC5B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC;YAAE,SAAS;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,MAAM,GAAG,UAAU,EAAE,CAAC;YACxB,UAAU,GAAG,MAAM,CAAC;YACpB,OAAO,GAAG,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC;QAClC,CAAC;aAAM,IAAI,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1D,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,IAAY;IACrD,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,IAAI,CAAC,GAAG,OAAO,CAAC;IAChB,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,GAAG,GAAG,IAAI,CAAC;QACX,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,IAAI,EAAE,GAAG,GAAG,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,EAAE,KAAK,GAAG;YAAE,EAAE,IAAI,IAAI,CAAC;;YACtB,EAAE,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IACD,IAAI,GAAG;QAAE,EAAE,IAAI,GAAG,CAAC;IACnB,OAAO,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,QAAQ,CAAC,EAAU;IAC1B,OAAO,EAAE,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC"}

package/dist/simhash.d.ts

@@ -0,0 +1,9 @@
+/**
+ * 64-bit SimHash over character 3-grams. Near-duplicate when Hamming ≤ 3.
+ * Implemented in-process so we never take a Python/datasketch dependency.
+ */
+export declare function simhash64(text: string): bigint;
+export declare function simhashHex(text: string): string;
+export declare function hamming64(a: bigint, b: bigint): number;
+export declare function nearDuplicate(aHex: string, bHex: string, maxDistance?: number): boolean;
+//# sourceMappingURL=simhash.d.ts.map

package/dist/simhash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"simhash.d.ts","sourceRoot":"","sources":["../src/simhash.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAgB9C;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQtD;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,SAAI,GAAG,OAAO,CAElF"}

package/dist/simhash.js

@@ -0,0 +1,58 @@
+/**
+ * 64-bit SimHash over character 3-grams. Near-duplicate when Hamming ≤ 3.
+ * Implemented in-process so we never take a Python/datasketch dependency.
+ */
+export function simhash64(text) {
+    const tokens = shingles(text.toLowerCase(), 3);
+    if (tokens.length === 0)
+        return 0n;
+    const acc = new Int32Array(64);
+    for (const token of tokens) {
+        const h = fnv1a64(token);
+        for (let i = 0; i < 64; i++) {
+            const bit = (h >> BigInt(i)) & 1n;
+            acc[i] = (acc[i] ?? 0) + (bit === 1n ? 1 : -1);
+        }
+    }
+    let out = 0n;
+    for (let i = 0; i < 64; i++) {
+        if ((acc[i] ?? 0) >= 0)
+            out |= 1n << BigInt(i);
+    }
+    return out;
+}
+export function simhashHex(text) {
+    return simhash64(text).toString(16).padStart(16, "0");
+}
+export function hamming64(a, b) {
+    let x = a ^ b;
+    let n = 0;
+    while (x) {
+        x &= x - 1n;
+        n++;
+    }
+    return n;
+}
+export function nearDuplicate(aHex, bHex, maxDistance = 3) {
+    return hamming64(BigInt(`0x${aHex}`), BigInt(`0x${bHex}`)) <= maxDistance;
+}
+function shingles(text, n) {
+    const cleaned = text.replace(/\s+/g, " ").trim();
+    if (cleaned.length < n)
+        return cleaned ? [cleaned] : [];
+    const out = [];
+    for (let i = 0; i <= cleaned.length - n; i++) {
+        out.push(cleaned.slice(i, i + n));
+    }
+    return out;
+}
+function fnv1a64(str) {
+    let hash = 0xcbf29ce484222325n;
+    const prime = 0x100000001b3n;
+    for (let i = 0; i < str.length; i++) {
+        hash ^= BigInt(str.charCodeAt(i));
+        hash = (hash * prime) & 0xffffffffffffffffn;
+    }
+    return hash;
+}
+//# sourceMappingURL=simhash.js.map

package/dist/simhash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"simhash.js","sourceRoot":"","sources":["../src/simhash.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YAClC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YAAE,GAAG,IAAI,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,CAAS,EAAE,CAAS;IAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,EAAE,CAAC;QACT,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACZ,CAAC,EAAE,CAAC;IACN,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,IAAY,EAAE,WAAW,GAAG,CAAC;IACvE,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,WAAW,CAAC;AAC5E,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,CAAS;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC1B,IAAI,IAAI,GAAG,mBAAmB,CAAC;IAC/B,MAAM,KAAK,GAAG,cAAc,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,mBAAmB,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/sitemap.d.ts

@@ -0,0 +1,3 @@
+import type { ParsedSitemap } from "./types.js";
+export declare function parseSitemapXml(raw: Buffer, sitemapUrl: string, gzipped: boolean): Promise<Omit<ParsedSitemap, "statusCode" | "contentType">>;
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../src/sitemap.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAK5D,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,GAAG,aAAa,CAAC,CAAC,CAa5D"}

package/dist/sitemap.js

@@ -0,0 +1,165 @@
+import { SaxesParser } from "saxes";
+import zlib from "node:zlib";
+import { promisify } from "node:util";
+import { absolutize } from "./url.js";
+const gunzip = promisify(zlib.gunzip);
+const NS = "http://www.sitemaps.org/schemas/sitemap/0.9";
+export async function parseSitemapXml(raw, sitemapUrl, gzipped) {
+    let bytes = raw;
+    let gzipBroken = false;
+    if (gzipped || sitemapUrl.endsWith(".gz")) {
+        try {
+            bytes = await gunzip(raw);
+        }
+        catch {
+            gzipBroken = true;
+            return empty(sitemapUrl, "gzip_broken", gzipBroken, raw.length);
+        }
+    }
+    const xml = bytes.toString("utf8");
+    return parseXml(xml, sitemapUrl, bytes.length, gzipBroken);
+}
+function empty(url, error, gzipBroken, bytes) {
+    return {
+        url,
+        error,
+        bytes,
+        gzipBroken,
+        malformed: true,
+        namespace: null,
+        isIndex: false,
+        nestedIndex: false,
+        urls: [],
+        childSitemaps: [],
+        relativeLocs: [],
+    };
+}
+function parseXml(xml, sitemapUrl, bytes, gzipBroken) {
+    const parser = new SaxesParser({ xmlns: true, fragment: false });
+    const urls = [];
+    const childSitemaps = [];
+    const relativeLocs = [];
+    let namespace = null;
+    let isIndex = false;
+    let nestedIndex = false;
+    let malformed = false;
+    let error = null;
+    let path = [];
+    let text = "";
+    let current = null;
+    let videoFields = {};
+    parser.on("error", (e) => {
+        malformed = true;
+        error = e.message;
+    });
+    parser.on("opentag", (tag) => {
+        const local = tag.local ?? tag.name.replace(/^.*:/, "");
+        path.push(local.toLowerCase());
+        text = "";
+        if (local.toLowerCase() === "urlset") {
+            namespace = tag.uri || tag.attributes?.xmlns?.value || NS;
+        }
+        if (local.toLowerCase() === "sitemapindex") {
+            isIndex = true;
+            namespace = tag.uri || tag.attributes?.xmlns?.value || NS;
+            if (path.filter((p) => p === "sitemapindex").length > 1)
+                nestedIndex = true;
+        }
+        if (local.toLowerCase() === "url") {
+            current = { images: [], hreflang: [] };
+            videoFields = {};
+        }
+    });
+    parser.on("text", (t) => {
+        text += t;
+    });
+    parser.on("closetag", (tag) => {
+        const local = (tag.local ?? tag.name.replace(/^.*:/, "")).toLowerCase();
+        const value = text.trim();
+        if (current) {
+            if (local === "loc" && path[path.length - 2] === "url")
+                current.loc = value;
+            if (local === "lastmod")
+                current.lastmod = value;
+            if (local === "changefreq")
+                current.changefreq = value;
+            if (local === "priority")
+                current.priority = value;
+            if (local === "loc" && path.includes("image"))
+                current.images.push(value);
+            if (local === "link") {
+                /* xhtml:link handled via attributes in opentag — see below */
+            }
+            if (path.includes("video")) {
+                videoFields[local] = value;
+            }
+            if (local === "url") {
+                const loc = current.loc ?? "";
+                if (loc && !/^https?:\/\//i.test(loc))
+                    relativeLocs.push(loc);
+                const videoValid = Object.keys(videoFields).length
+                    ? Boolean(videoFields["thumbnail_loc"] &&
+                        videoFields["title"] &&
+                        videoFields["description"] &&
+                        (videoFields["content_loc"] || videoFields["player_loc"]))
+                    : null;
+                urls.push({
+                    loc,
+                    lastmod: current.lastmod ?? null,
+                    changefreq: current.changefreq ?? null,
+                    priority: current.priority ?? null,
+                    images: current.images,
+                    videoValid,
+                    hreflang: current.hreflang,
+                });
+                current = null;
+            }
+        }
+        else if (isIndex && local === "loc") {
+            childSitemaps.push(value);
+        }
+        path.pop();
+        text = "";
+    });
+    // saxes xmlns attributes: capture xhtml:link hreflang on opentag via a second pass
+    const hrefLangRe = /<xhtml:link[^>]*rel=["']alternate["'][^>]*hreflang=["']([^"']+)["'][^>]*href=["']([^"']+)["'][^>]*\/?>/gi;
+    let hm;
+    const extras = [];
+    while ((hm = hrefLangRe.exec(xml))) {
+        extras.push({ lang: hm[1] ?? "", href: hm[2] ?? "" });
+    }
+    try {
+        parser.write(xml);
+        parser.close();
+    }
+    catch (e) {
+        malformed = true;
+        error = e instanceof Error ? e.message : "parse";
+    }
+    if (extras.length && urls.length) {
+        const first = urls[0];
+        if (first)
+            first.hreflang.push(...extras);
+    }
+    for (const u of urls) {
+        if (u.loc && !u.loc.startsWith("http")) {
+            const abs = absolutize(u.loc, sitemapUrl);
+            if (abs)
+                u.loc = abs;
+        }
+    }
+    return {
+        url: sitemapUrl,
+        error,
+        bytes,
+        gzipBroken,
+        malformed,
+        namespace,
+        isIndex,
+        nestedIndex,
+        urls,
+        childSitemaps,
+        relativeLocs,
+    };
+}
+//# sourceMappingURL=sitemap.js.map

package/dist/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../src/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AACpC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACtC,MAAM,EAAE,GAAG,6CAA6C,CAAC;AAEzD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAW,EACX,UAAkB,EAClB,OAAgB;IAEhB,IAAI,KAAK,GAAG,GAAG,CAAC;IAChB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,UAAU,GAAG,IAAI,CAAC;YAClB,OAAO,KAAK,CAAC,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,KAAK,CACZ,GAAW,EACX,KAAa,EACb,UAAmB,EACnB,KAAa;IAEb,OAAO;QACL,GAAG;QACH,KAAK;QACL,KAAK;QACL,UAAU;QACV,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,KAAK;QACd,WAAW,EAAE,KAAK;QAClB,IAAI,EAAE,EAAE;QACR,aAAa,EAAE,EAAE;QACjB,YAAY,EAAE,EAAE;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,GAAW,EACX,UAAkB,EAClB,KAAa,EACb,UAAmB;IAEnB,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,OAAO,GAKA,IAAI,CAAC;IAChB,IAAI,WAAW,GAA2B,EAAE,CAAC;IAE7C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;QACvB,SAAS,GAAG,IAAI,CAAC;QACjB,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/B,IAAI,GAAG,EAAE,CAAC;QACV,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE,CAAC;YACrC,SAAS,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,cAAc,EAAE,CAAC;YAC3C,OAAO,GAAG,IAAI,CAAC;YACf,SAAS,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,WAAW,GAAG,IAAI,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,KAAK,EAAE,CAAC;YAClC,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YACvC,WAAW,GAAG,EAAE,CAAC;QACnB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;QACtB,IAAI,IAAI,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;QAC5B,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QACxE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,KAAK;gBAAE,OAAO,CAAC,GAAG,GAAG,KAAK,CAAC;YAC5E,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC;YACjD,IAAI,KAAK,KAAK,YAAY;gBAAE,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC;YACvD,IAAI,KAAK,KAAK,UAAU;gBAAE,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;YACnD,IAAI,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1E,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;gBACrB,8DAA8D;YAChE,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,WAAW,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;YAC7B,CAAC;YACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;gBAC9B,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM;oBAChD,CAAC,CAAC,OAAO,CACL,WAAW,CAAC,eAAe,CAAC;wBAC5B,WAAW,CAAC,OAAO,CAAC;wBACpB,WAAW,CAAC,aAAa,CAAC;wBAC1B,CAAC,WAAW,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC,CAAC,CAC1D;oBACH,CAAC,CAAC,IAAI,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC;oBACR,GAAG;oBACH,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;oBAChC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;oBACtC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;oBAClC,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,UAAU;oBACV,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B,CAAC,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,IAAI,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,mFAAmF;IACnF,MAAM,UAAU,GACd,0GAA0G,CAAC;IAC7G,IAAI,EAA0B,CAAC;IAC/B,MAAM,MAAM,GAAqC,EAAE,CAAC;IACpD,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,SAAS,GAAG,IAAI,CAAC;QACjB,KAAK,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACnD,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,KAAK;YAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAC1C,IAAI,GAAG;gBAAE,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO;QACL,GAAG,EAAE,UAAU;QACf,KAAK;QACL,KAAK;QACL,UAAU;QACV,SAAS;QACT,SAAS;QACT,OAAO;QACP,WAAW;QACX,IAAI;QACJ,aAAa;QACb,YAAY;KACb,CAAC;AACJ,CAAC"}