nfunc-mcp 0.3.0 → 0.5.0

package/dist/utils/sitemapReader.js

@@ -0,0 +1,272 @@
+/**
+ * Sitemap discovery.
+ *
+ * The cheapest honest way to learn which pages a site has. A sitemap is
+ * authoritative (the site published it), instant (one or two requests), and
+ * needs no crawling, no robots.txt politeness budget and no HTML parsing. A
+ * crawler is the fallback for sites without one, and is deliberately not built
+ * yet — everything downstream works without it.
+ *
+ * Sitemaps are parsed with regular expressions rather than an XML library.
+ * That is normally a mistake, but a sitemap is a machine-generated document
+ * with a fixed two-element vocabulary, and the alternative is a dependency
+ * carried by every install of this server for one code path. The parser reads
+ * <loc> only and ignores everything else, so malformed markup degrades to
+ * fewer URLs rather than to wrong ones.
+ */
+import { gunzipSync } from "node:zlib";
+import { httpGet } from "./httpClient.js";
+/** A sitemap index can point at hundreds of children; walking all of them is rarely worth it. */
+const MAX_SITEMAPS = 25;
+const MAX_URLS = 50_000;
+const FETCH_TIMEOUT_MS = 20_000;
+const LOC_RE = /<loc>\s*([^<\s][^<]*?)\s*<\/loc>/gi;
+function decodeXmlEntities(value) {
+    return value
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'")
+        .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d)))
+        // Ampersand last, so "&amp;lt;" does not become "<".
+        .replace(/&amp;/g, "&");
+}
+function extractLocs(xml) {
+    const out = [];
+    for (const match of xml.matchAll(LOC_RE)) {
+        const loc = decodeXmlEntities(match[1].trim());
+        if (loc)
+            out.push(loc);
+    }
+    return out;
+}
+/** A <sitemapindex> points at more sitemaps; a <urlset> holds pages. */
+function isSitemapIndex(xml) {
+    return /<sitemapindex[\s>]/i.test(xml);
+}
+/**
+ * Sitemap locations declared in robots.txt.
+ *
+ * Large sites frequently do not serve /sitemap.xml and only announce the real
+ * location here, so checking robots first turns a "no sitemap found" dead end
+ * into a hit. It is also the polite thing to read before touching a site.
+ */
+async function sitemapsFromRobots(origin) {
+    const robotsUrl = new URL("/robots.txt", origin).toString();
+    const result = await httpGet(robotsUrl, { timeoutMs: 10_000, retries: 1 });
+    if (!result.ok)
+        return [];
+    const found = [];
+    for (const line of result.body.split(/\r?\n/)) {
+        const match = /^\s*sitemap:\s*(\S+)/i.exec(line);
+        if (match)
+            found.push(match[1]);
+    }
+    return found;
+}
+/**
+ * Non-HTML entries a performance audit should never spend a PSI call on.
+ * Image and video sitemaps are common and would otherwise fill the sample.
+ */
+function looksAuditable(url) {
+    return !/\.(jpe?g|png|gif|webp|avif|svg|ico|pdf|zip|gz|mp4|webm|mp3|xml|json|txt|css|js)(\?|$)/i.test(url);
+}
+/**
+ * Sitemap locations worth guessing, beyond the two standard ones.
+ *
+ * Ordered by how often they pay off. These only run when the standard
+ * locations and robots.txt have produced nothing, so the common case still
+ * costs one or two requests — a 404 from Google's edge is cheap, but seven of
+ * them on every audit would not be.
+ */
+const CANDIDATE_PATHS = [
+    "/sitemap-index.xml",
+    "/sitemap/sitemap.xml",
+    "/sitemap/index.xml",
+    "/wp-sitemap.xml", // WordPress 5.5+
+    "/sitemap_index.xml.gz",
+    "/sitemap1.xml",
+    "/sitemap.txt", // plain text, one URL per line
+];
+/**
+ * `<link rel="sitemap">` in the homepage head.
+ *
+ * Rare but authoritative when present, and it costs one request we can often
+ * justify anyway. Only consulted after the cheaper guesses fail.
+ */
+async function sitemapFromHomepageLink(origin) {
+    const result = await httpGet(origin, { timeoutMs: 15_000, retries: 0 });
+    if (!result.ok)
+        return [];
+    const found = [];
+    for (const match of result.body.matchAll(/<link\b[^>]*>/gi)) {
+        const tag = match[0];
+        if (!/rel=["']?sitemap["']?/i.test(tag))
+            continue;
+        const href = /href=["']([^"']+)["']/i.exec(tag)?.[1];
+        if (href) {
+            try {
+                found.push(new URL(href, origin).toString());
+            }
+            catch {
+                // Unparseable href — ignore rather than fail discovery over it.
+            }
+        }
+    }
+    return found;
+}
+/**
+ * A sitemap.txt is a bare newline-delimited URL list, not XML. Detected by
+ * content rather than extension, because servers are inconsistent about both.
+ */
+function isPlainTextSitemap(body) {
+    const head = body.trimStart().slice(0, 200);
+    return !head.startsWith("<") && /^https?:\/\//im.test(head);
+}
+function extractPlainTextUrls(body) {
+    return body
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => /^https?:\/\//i.test(line));
+}
+export async function readSitemap(origin) {
+    const warnings = [];
+    const sitemapsRead = [];
+    const attempted = [];
+    const urls = new Set();
+    const seen = new Set();
+    let truncated = false;
+    /**
+     * Tiers, cheapest and most authoritative first. Each is tried only if every
+     * earlier one came up empty, so a site that declares its sitemap properly
+     * still costs two requests while a site that hides it gets a real search.
+     */
+    const declared = await sitemapsFromRobots(origin);
+    const tiers = [
+        { name: "robots.txt", locations: async () => declared },
+        {
+            name: "standard locations",
+            locations: async () => [
+                new URL("/sitemap.xml", origin).toString(),
+                new URL("/sitemap_index.xml", origin).toString(),
+            ],
+        },
+        { name: "homepage <link rel=sitemap>", locations: () => sitemapFromHomepageLink(origin) },
+        {
+            name: "common CMS locations",
+            locations: async () => CANDIDATE_PATHS.map((path) => new URL(path, origin).toString()),
+        },
+    ];
+    for (const tier of tiers) {
+        // Stop on the first tier that finds a sitemap *document*, not the first
+        // that yields URLs. web.dev serves a valid index whose children time out;
+        // falling through on an empty result sent discovery guessing at nine more
+        // locations for 43s when the sitemap had already been located.
+        if (sitemapsRead.length > 0)
+            break;
+        const queue = await tier.locations();
+        let consecutiveFailures = 0;
+        while (queue.length > 0) {
+            if (sitemapsRead.length >= MAX_SITEMAPS || urls.size >= MAX_URLS) {
+                truncated = true;
+                break;
+            }
+            const next = queue.shift();
+            if (seen.has(next))
+                continue;
+            seen.add(next);
+            attempted.push(next);
+            // Large sites commonly serve .xml.gz. It arrives as an opaque gzip payload
+            // rather than with Content-Encoding, so fetch does not inflate it and a
+            // text read produces binary noise — hence the explicit binary path.
+            const gzipped = /\.gz(\?|$)/i.test(next);
+            // No retry: discovery must stay fast, and a sitemap that times out once
+            // usually times out again. Retrying doubled the cost of the slow case
+            // for no observed benefit.
+            const result = await httpGet(next, {
+                timeoutMs: FETCH_TIMEOUT_MS,
+                retries: 0,
+                raw: gzipped,
+            });
+            if (!result.ok) {
+                // A 404 or 403 on a guessed location is the expected answer to "is it
+                // here?" — cheap, informative, and no reason to stop guessing. Only
+                // expensive failures (timeouts, 5xx) count toward giving up.
+                if (result.status === 404 || result.status === 403)
+                    continue;
+                consecutiveFailures++;
+                // A sitemap index can list hundreds of children. If the first few all
+                // time out, the rest almost certainly will too, at a full timeout each.
+                if (consecutiveFailures >= 3) {
+                    warnings.push(`Gave up on ${tier.name} after ${consecutiveFailures} consecutive failures ` +
+                        `(last: ${result.error ?? `HTTP ${result.status}`}).`);
+                    break;
+                }
+                warnings.push(`Could not read ${next}: ${result.error ?? `HTTP ${result.status}`}`);
+                continue;
+            }
+            consecutiveFailures = 0;
+            let body = result.body;
+            if (gzipped) {
+                try {
+                    body = gunzipSync(Buffer.from(result.bytes ?? new Uint8Array())).toString("utf8");
+                }
+                catch (err) {
+                    warnings.push(`Could not decompress ${next}: ${err.message}`);
+                    continue;
+                }
+            }
+            if (isPlainTextSitemap(body)) {
+                const plain = extractPlainTextUrls(body).filter(looksAuditable);
+                if (plain.length === 0)
+                    continue;
+                sitemapsRead.push(next);
+                for (const loc of plain) {
+                    if (urls.size >= MAX_URLS) {
+                        truncated = true;
+                        break;
+                    }
+                    urls.add(loc);
+                }
+                continue;
+            }
+            const locs = extractLocs(body);
+            if (locs.length === 0)
+                continue;
+            sitemapsRead.push(next);
+            if (isSitemapIndex(body)) {
+                for (const child of locs)
+                    if (!seen.has(child))
+                        queue.push(child);
+                continue;
+            }
+            for (const loc of locs) {
+                if (urls.size >= MAX_URLS) {
+                    truncated = true;
+                    break;
+                }
+                if (looksAuditable(loc))
+                    urls.add(loc);
+            }
+        }
+    }
+    if (sitemapsRead.length > 0 && urls.size === 0) {
+        warnings.push(`Found a sitemap at ${sitemapsRead[0]} but could not extract any URLs from it — ` +
+            `its child documents failed to load or contained no page entries. Supply URLs ` +
+            `with discovery:"list" or discovery:"csv".`);
+    }
+    if (sitemapsRead.length === 0) {
+        warnings.push(`No sitemap found for ${origin}. Tried ${attempted.length} location(s): ` +
+            `robots.txt, the standard paths, the homepage <link rel="sitemap">, and ` +
+            `common CMS locations. ` +
+            `Supply URLs directly with discovery:"list", or point at a CSV export with ` +
+            `discovery:"csv" — an analytics top-pages export is the better input for a ` +
+            `performance audit anyway, since it is weighted by real traffic.`);
+    }
+    if (truncated) {
+        warnings.push(`Stopped after ${sitemapsRead.length} sitemap documents and ${urls.size} URLs. ` +
+            `The template breakdown below is based on that subset.`);
+    }
+    return { urls: [...urls], sitemapsRead, truncated, attempted, warnings };
+}
+//# sourceMappingURL=sitemapReader.js.map

package/dist/utils/sitemapReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemapReader.js","sourceRoot":"","sources":["../../src/utils/sitemapReader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAa1C,iGAAiG;AACjG,MAAM,YAAY,GAAG,EAAE,CAAC;AACxB,MAAM,QAAQ,GAAG,MAAM,CAAC;AACxB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,GAAG,oCAAoC,CAAC;AAEpD,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,qDAAqD;SACpD,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wEAAwE;AACxE,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,MAAc;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,CAAC,wFAAwF,CAAC,IAAI,CACnG,GAAG,CACJ,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG;IACtB,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB;IACpB,iBAAiB,EAAS,iBAAiB;IAC3C,uBAAuB;IACvB,eAAe;IACf,cAAc,EAAY,+BAA+B;CAC1D,CAAC;AAEF;;;;;GAKG;AACH,KAAK,UAAU,uBAAuB,CAAC,MAAc;IACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAClD,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,IAAI;SACR,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc;IAC9C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB;;;;OAIG;IACH,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,KAAK,GAAgE;QACzE,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,QAAQ,EAAE;QACvD;YACE,IAAI,EAAE,oBAAoB;YAC1B,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC;gBACrB,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE;gBAC1C,IAAI,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE;aACjD;SACF;QACD,EAAE,IAAI,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE;QACzF;YACE,IAAI,EAAE,sBAAsB;YAC5B,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;SACvF;KACF,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,wEAAwE;QACxE,0EAA0E;QAC1E,0EAA0E;QAC1E,+DAA+D;QAC/D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM;QAEnC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;gBACjE,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAY,CAAC;YACrC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAErB,2EAA2E;YAC3E,wEAAwE;YACxE,oEAAoE;YACpE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,wEAAwE;YACxE,sEAAsE;YACtE,2BAA2B;YAC3B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;gBACjC,SAAS,EAAE,gBAAgB;gBAC3B,OAAO,EAAE,CAAC;gBACV,GAAG,EAAE,OAAO;aACb,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,sEAAsE;gBACtE,oEAAoE;gBACpE,6DAA6D;gBAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;oBAAE,SAAS;gBAE7D,mBAAmB,EAAE,CAAC;gBACtB,sEAAsE;gBACtE,wEAAwE;gBACxE,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;oBAC7B,QAAQ,CAAC,IAAI,CACX,cAAc,IAAI,CAAC,IAAI,UAAU,mBAAmB,wBAAwB;wBAC1E,UAAU,MAAM,CAAC,KAAK,IAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,IAAI,CACxD,CAAC;oBACF,MAAM;gBACR,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,kBAAkB,IAAI,KAAK,MAAM,CAAC,KAAK,IAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,mBAAmB,GAAG,CAAC,CAAC;YAExB,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACH,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACpF,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,QAAQ,CAAC,IAAI,CAAC,wBAAwB,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;oBACzE,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBACjC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;wBAC1B,SAAS,GAAG,IAAI,CAAC;wBACjB,MAAM;oBACR,CAAC;oBACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAChC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAExB,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,KAAK,IAAI,IAAI;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;wBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClE,SAAS;YACX,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC1B,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM;gBACR,CAAC;gBACD,IAAI,cAAc,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CACX,sBAAsB,YAAY,CAAC,CAAC,CAAC,4CAA4C;YAC/E,+EAA+E;YAC/E,2CAA2C,CAC9C,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CACX,wBAAwB,MAAM,WAAW,SAAS,CAAC,MAAM,gBAAgB;YACvE,yEAAyE;YACzE,wBAAwB;YACxB,4EAA4E;YAC5E,4EAA4E;YAC5E,iEAAiE,CACpE,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CACX,iBAAiB,YAAY,CAAC,MAAM,0BAA0B,IAAI,CAAC,IAAI,SAAS;YAC9E,uDAAuD,CAC1D,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC3E,CAAC"}

package/dist/utils/urlClassifier.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Cluster a URL list into page templates.
+ *
+ * A performance audit's unit of analysis is the template, not the URL. Nobody
+ * needs 3,904 product pages measured; they need to know what a product page
+ * costs. The manual Five Below audit was organised exactly this way — Homepage,
+ * PLP (category), PLP (subcategory), PLP (paginated), PDP, Cart, Search,
+ * Info — with representative URLs sampled per template, and this reproduces
+ * that structure automatically instead of asking someone to hand-write the
+ * list.
+ *
+ * The clustering is a heuristic and will occasionally be wrong. That is
+ * acceptable because the plan tool shows its work and the user approves the
+ * sample before any PSI call is spent: a misclassification costs a
+ * conversation turn, not quota.
+ */
+export interface UrlTemplate {
+    id: string;
+    label: string;
+    /** Path shape with high-cardinality segments collapsed, e.g. "/categories/*". */
+    pattern: string;
+    urlCount: number;
+    /** How many URLs to audit, given the population size. */
+    suggestedSample: number;
+    /** Representative URLs, most-canonical first. */
+    candidates: string[];
+    auditable: boolean;
+    reason?: string;
+    recommendation?: string;
+    /**
+     * True when *any* URL in the template is session-gated — computed over the
+     * whole group, not the sampled candidates. Deciding this from candidates made
+     * the verdict depend on which URLs the sampler happened to pick, so a
+     * template containing /cart could pass as auditable purely by luck.
+     */
+    anySessionGated: boolean;
+}
+export interface ClassifyOptions {
+    /** Patterns matching fewer URLs than this are folded into "Other". */
+    minTemplateSize?: number;
+}
+export declare function classifyUrls(rawUrls: string[], options?: ClassifyOptions): {
+    templates: UrlTemplate[];
+    skipped: number;
+};

package/dist/utils/urlClassifier.js

@@ -0,0 +1,267 @@
+import { isSessionGated } from "./publicUrl.js";
+/**
+ * Query parameters that identify the visitor or the campaign rather than the
+ * page. Left in, they would split one template into dozens of "shapes" that
+ * all render identically.
+ */
+const TRACKING_PARAMS = /^(utm_|gclid|fbclid|msclkid|mc_cid|mc_eid|_ga|igshid|srsltid|ref|referrer|source|cmpid|campaign)/i;
+/**
+ * A segment position needs at least this many distinct values to be wildcarded.
+ * Two siblings are as likely to be two distinct pages as one template with two
+ * instances, so the floor is three.
+ */
+const WILDCARD_DISTINCT = 3;
+/**
+ * The signal that separates an identifier from a section name: **fan-out**.
+ *
+ * Cardinality alone does not work, and getting this wrong is visible on real
+ * sites. On nodejs.org, "/en/{blog,download,learn,about}" has only a handful of
+ * distinct values, so a pure count threshold wildcarded it and merged 1,054
+ * blog posts, download pages and API docs into one meaningless "/en/*\/*\/*"
+ * template.
+ *
+ * A section name is shared by many URLs (blog → 900 pages); an identifier
+ * belongs to one (v04.3 → 1 page). So a segment is a wildcard when its values
+ * each account for very few URLs, and stays literal when each value heads a
+ * substantial subtree — regardless of how many there are.
+ */
+const MAX_MEAN_FANOUT = 2.5;
+/**
+ * The first path segment is the site's own taxonomy — "categories", "products",
+ * "info" — and collapsing it produces the useless pattern "/*". It only gets
+ * wildcarded on a clear id explosion.
+ */
+const WILDCARD_DISTINCT_ROOT = 20;
+const ID_LIKE = /^(\d+|[0-9a-f]{8,}|[0-9a-f-]{16,}|p\d+|sku[-_]?\w+)$/i;
+/**
+ * Route vocabulary — segments that name a *kind* of page, never an instance.
+ *
+ * Fan-out alone misreads these on a small URL list. Given eight URLs under
+ * /ecommerce/, the segments {page, product, cart, checkout, my-account} average
+ * 1.6 URLs each, which looks exactly like a set of identifiers — so the
+ * classifier collapsed paginated listings and product detail pages into one
+ * "/ecommerce/*\/*" template and proposed sampling them as if they were the
+ * same page type. On the full inventory the fan-out for "product" would be
+ * large and the heuristic would work, but a plan should not be wrong just
+ * because the input list is short.
+ */
+const ROUTE_WORDS = new Set([
+    "page", "pages", "product", "products", "category", "categories", "collection",
+    "collections", "cart", "basket", "checkout", "account", "my-account", "login",
+    "signin", "register", "search", "tag", "tags", "author", "feed", "blog", "news",
+    "article", "articles", "post", "posts", "info", "about", "help", "support",
+    "shop", "store", "browse", "dept", "department", "item", "items", "sku",
+]);
+/**
+ * A content slug: hyphenated and long enough to be a title rather than a route.
+ * "adrienne-trek-jacket" is an instance; "my-account" is a route.
+ */
+function looksLikeSlug(value) {
+    if (!value.includes("-"))
+        return false;
+    const words = value.split("-").length;
+    return value.length >= 14 || words >= 3;
+}
+function parseUrl(raw) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        return null;
+    }
+    const segments = url.pathname.split("/").filter(Boolean);
+    const keys = [...url.searchParams.keys()]
+        .filter((k) => !TRACKING_PARAMS.test(k))
+        .map((k) => k.toLowerCase())
+        .sort();
+    return { raw, segments, querySig: [...new Set(keys)].join("&") };
+}
+/**
+ * Wildcard a segment when its values are identifiers rather than taxonomy.
+ * Cardinality alone is the main signal; an id-shaped majority is enough on its
+ * own, since "/p/12345" and "/p/12346" are the same page template even when
+ * only two exist.
+ */
+function shouldWildcard(depth, values) {
+    const distinct = new Set(values);
+    if (distinct.size < 2)
+        return false;
+    // Numeric, hash-shaped and long-slug values are identifiers whatever their
+    // fan-out; route vocabulary never is.
+    const values_ = [...distinct].map((v) => v.toLowerCase());
+    const idLike = values_.filter((v) => ID_LIKE.test(v) || looksLikeSlug(v)).length;
+    if (idLike / distinct.size > 0.6)
+        return true;
+    if (values_.every((v) => ROUTE_WORDS.has(v)))
+        return false;
+    if (depth === 0 && distinct.size < WILDCARD_DISTINCT_ROOT)
+        return false;
+    if (distinct.size < WILDCARD_DISTINCT)
+        return false;
+    return values.length / distinct.size <= MAX_MEAN_FANOUT;
+}
+/**
+ * Recursive descent over path segments.
+ *
+ * At each depth, URLs that end here form their own pattern, and the rest are
+ * grouped by their next segment — wildcarded together when that segment looks
+ * like an identifier, kept apart when it looks like a section name. Recursion
+ * (rather than one global pass per depth) is what keeps "/categories/*" and
+ * "/products/*" separate instead of collapsing both into "/*\/*".
+ */
+function descend(urls, depth, prefix, out) {
+    const terminal = urls.filter((u) => u.segments.length === depth);
+    const deeper = urls.filter((u) => u.segments.length > depth);
+    if (terminal.length > 0) {
+        const key = "/" + prefix.join("/");
+        for (const u of terminal) {
+            const full = u.querySig ? `${key}?${u.querySig}` : key;
+            const bucket = out.get(full);
+            if (bucket)
+                bucket.push(u);
+            else
+                out.set(full, [u]);
+        }
+    }
+    if (deeper.length === 0)
+        return;
+    if (shouldWildcard(depth, deeper.map((u) => u.segments[depth]))) {
+        descend(deeper, depth + 1, [...prefix, "*"], out);
+        return;
+    }
+    const groups = new Map();
+    for (const u of deeper) {
+        const token = u.segments[depth];
+        const bucket = groups.get(token);
+        if (bucket)
+            bucket.push(u);
+        else
+            groups.set(token, [u]);
+    }
+    for (const [token, group] of groups) {
+        descend(group, depth + 1, [...prefix, token], out);
+    }
+}
+/**
+ * Human labels for common commerce and content shapes.
+ *
+ * Deliberately conservative: an unrecognised pattern is labelled by its path
+ * rather than guessed at, because a wrong label in an executive report is
+ * worse than a literal one.
+ */
+function labelFor(pattern) {
+    const [path, query] = pattern.split("?");
+    const p = path.toLowerCase();
+    const depth = path.split("/").filter(Boolean).length;
+    const wildcards = (path.match(/\*/g) ?? []).length;
+    const paginated = query ? ` (${query.includes("p") || query.includes("page") ? "paginated" : "filtered"})` : "";
+    if (path === "/")
+        return "Homepage";
+    if (/(^|\/)(cart|basket|bag)(\/|$)/.test(p))
+        return "Cart";
+    if (/(^|\/)(checkout|payment)(\/|$)/.test(p))
+        return "Checkout";
+    if (/(^|\/)(search|s|find)(\/|$)/.test(p))
+        return "Search results";
+    if (/(^|\/)(store-locator|stores|locations|find-a-store)(\/|$)/.test(p))
+        return "Store locator";
+    if (/(^|\/)(account|my-account|profile|orders|wishlist)(\/|$)/.test(p))
+        return "Account";
+    if (/(^|\/)(login|signin|sign-in|register|signup)(\/|$)/.test(p))
+        return "Login / register";
+    if (/(^|\/)(product|products|p|item|items|sku|dp)(\/|$)/.test(p))
+        return `PDP (product)${paginated}`;
+    if (/(^|\/)(blog|news|articles|stories|posts)(\/|$)/.test(p))
+        return `Content / article${paginated}`;
+    if (/(^|\/)(info|about|help|faq|support|legal|privacy|terms|policies|customer-service)(\/|$)/.test(p))
+        return `Info / static${paginated}`;
+    if (/(^|\/)(categories|category|c|collections|shop|browse|dept|department)(\/|$)/.test(p)) {
+        const kind = wildcards >= 2 ? "subcategory" : "category";
+        return `PLP (${kind})${paginated}`;
+    }
+    if (wildcards > 0 && depth <= 2)
+        return `Listing${paginated}`;
+    return `${path}${paginated}`;
+}
+/**
+ * Sample size by population.
+ *
+ * Three is the ceiling because PSI is slow and quota is finite, and because
+ * variance between pages of one template is usually smaller than variance
+ * between templates — a fourth PDP tells you much less than a first Cart.
+ */
+function suggestSample(population) {
+    if (population <= 1)
+        return 1;
+    if (population <= 5)
+        return 2;
+    return 3;
+}
+/**
+ * Representatives, chosen for spread rather than at random.
+ *
+ * Shortest first: it is usually the canonical, most-linked instance of the
+ * template. Then the longest and a middle one, because URL length correlates
+ * with depth and with how much content the page carries, and a template's
+ * worst page is more often its longest.
+ */
+function pickCandidates(urls, count) {
+    const sorted = [...urls].sort((a, b) => a.length - b.length || a.localeCompare(b));
+    if (sorted.length <= count)
+        return sorted;
+    const picks = [sorted[0]];
+    if (count >= 2)
+        picks.push(sorted[sorted.length - 1]);
+    if (count >= 3)
+        picks.push(sorted[Math.floor(sorted.length / 2)]);
+    for (let i = 1; picks.length < count && i < sorted.length; i++) {
+        if (!picks.includes(sorted[i]))
+            picks.push(sorted[i]);
+    }
+    return picks.slice(0, count);
+}
+function slugify(pattern) {
+    return (pattern
+        .replace(/[?&=]/g, "-")
+        .replace(/\*/g, "wild")
+        .replace(/[^a-z0-9]+/gi, "-")
+        .replace(/^-+|-+$/g, "")
+        .toLowerCase() || "root");
+}
+export function classifyUrls(rawUrls, options = {}) {
+    const { minTemplateSize = 1 } = options;
+    const parsed = [];
+    let skipped = 0;
+    for (const raw of rawUrls) {
+        const p = parseUrl(raw);
+        if (p)
+            parsed.push(p);
+        else
+            skipped++;
+    }
+    const buckets = new Map();
+    descend(parsed, 0, [], buckets);
+    const templates = [];
+    for (const [pattern, group] of buckets) {
+        if (group.length < minTemplateSize)
+            continue;
+        const urls = group.map((g) => g.raw);
+        const suggested = suggestSample(urls.length);
+        templates.push({
+            id: slugify(pattern),
+            label: labelFor(pattern),
+            pattern,
+            urlCount: urls.length,
+            suggestedSample: suggested,
+            candidates: pickCandidates(urls, suggested),
+            auditable: true,
+            anySessionGated: urls.some(isSessionGated),
+        });
+    }
+    // Biggest templates first: a reader scanning the plan should see the shapes
+    // that dominate the site before the long tail of one-off pages.
+    templates.sort((a, b) => b.urlCount - a.urlCount || a.pattern.localeCompare(b.pattern));
+    return { templates, skipped };
+}
+//# sourceMappingURL=urlClassifier.js.map

package/dist/utils/urlClassifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"urlClassifier.js","sourceRoot":"","sources":["../../src/utils/urlClassifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA+ChD;;;;GAIG;AACH,MAAM,eAAe,GAAG,mGAAmG,CAAC;AAE5H;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;;;;;;;;;;;GAaG;AACH,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC,MAAM,OAAO,GAAG,uDAAuD,CAAC;AAExE;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IAC1B,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY;IAC9E,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO;IAC7E,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC/E,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAC1E,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;CACxE,CAAC,CAAC;AAEH;;;GAGG;AACH,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,KAAK,CAAC,MAAM,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;SACtC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;SAC3B,IAAI,EAAE,CAAC;IACV,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACnE,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAgB;IACrD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpC,2EAA2E;IAC3E,sCAAsC;IACtC,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACjF,IAAI,MAAM,GAAG,QAAQ,CAAC,IAAI,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3D,IAAI,KAAK,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,GAAG,sBAAsB;QAAE,OAAO,KAAK,CAAC;IACxE,IAAI,QAAQ,CAAC,IAAI,GAAG,iBAAiB;QAAE,OAAO,KAAK,CAAC;IAEpD,OAAO,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,IAAI,IAAI,eAAe,CAAC;AAC1D,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,OAAO,CACd,IAAiB,EACjB,KAAa,EACb,MAAgB,EAChB,GAA6B;IAE7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IAE7D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YACvD,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;gBACtB,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEhC,IAAI,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,OAAO,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QAClD,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YACtB,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,SAAS,QAAQ,CAAC,OAAe;IAC/B,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACrD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhH,IAAI,IAAI,KAAK,GAAG;QAAE,OAAO,UAAU,CAAC;IACpC,IAAI,+BAA+B,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IAC3D,IAAI,gCAAgC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,UAAU,CAAC;IAChE,IAAI,6BAA6B,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,gBAAgB,CAAC;IACnE,IAAI,2DAA2D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,eAAe,CAAC;IAChG,IAAI,0DAA0D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzF,IAAI,oDAAoD,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5F,IAAI,oDAAoD,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,gBAAgB,SAAS,EAAE,CAAC;IACrG,IAAI,gDAAgD,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,oBAAoB,SAAS,EAAE,CAAC;IACrG,IAAI,yFAAyF,CAAC,IAAI,CAAC,CAAC,CAAC;QACnG,OAAO,gBAAgB,SAAS,EAAE,CAAC;IACrC,IAAI,6EAA6E,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC;QACzD,OAAO,QAAQ,IAAI,IAAI,SAAS,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,UAAU,SAAS,EAAE,CAAC;IAC9D,OAAO,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;AAC/B,CAAC;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,UAAkB;IACvC,IAAI,UAAU,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,IAAI,UAAU,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,IAAc,EAAE,KAAa;IACnD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,IAAI,MAAM,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1C,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,KAAK,IAAI,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtD,IAAI,KAAK,IAAI,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/D,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,OAAO,CAAC,OAAe;IAC9B,OAAO,CACL,OAAO;SACJ,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,WAAW,EAAE,IAAI,MAAM,CAC3B,CAAC;AACJ,CAAC;AAOD,MAAM,UAAU,YAAY,CAC1B,OAAiB,EACjB,UAA2B,EAAE;IAE7B,MAAM,EAAE,eAAe,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAExC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YACjB,OAAO,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe;YAAE,SAAS;QAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC;YACpB,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC;YACxB,OAAO;YACP,QAAQ,EAAE,IAAI,CAAC,MAAM;YACrB,eAAe,EAAE,SAAS;YAC1B,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC;YAC3C,SAAS,EAAE,IAAI;YACf,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,gEAAgE;IAChE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACxF,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC"}

package/dist/utils/urlInput.d.ts

@@ -0,0 +1,30 @@
+/**
+ * One input, three shapes.
+ *
+ * People arrive with a URL, a list they pasted from a spreadsheet, or a CSV
+ * export. Making them say which is which is a tax on the common case, and an
+ * agent relaying a user's request should not have to guess a parameter name.
+ * So the `url` input accepts any of the three and this module works out what it
+ * was given.
+ *
+ * Detection is by shape, not by a flag, and deliberately conservative: anything
+ * ambiguous is reported rather than guessed at, because silently auditing the
+ * wrong set of pages is worse than an error.
+ */
+export type UrlInputKind = "single" | "list" | "csv";
+export interface ResolvedUrls {
+    urls: string[];
+    kind: UrlInputKind;
+    /** Where a CSV was read from, or the column used. Empty for the other kinds. */
+    source?: string;
+    warnings: string[];
+}
+/**
+ * Resolve whatever was handed in into a URL list.
+ *
+ * Order matters: the CSV check runs first because a path can contain no commas
+ * and no scheme, which would otherwise be coerced into `https://./file.csv`.
+ */
+export declare function resolveUrlInput(input: string): ResolvedUrls;
+/** Accept an explicit array too, for callers that already have one. */
+export declare function resolveUrlInputs(input: string | undefined, urls: string[] | undefined): ResolvedUrls;