nfunc-mcp 0.3.0 → 0.4.0

package/dist/utils/publicUrl.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Whether PSI can reach a URL at all.
+ *
+ * PSI fetches the target from Google's own infrastructure, so "can I open this
+ * in my browser" is not the test — the page has to be reachable from the
+ * public internet. Localhost and RFC1918 addresses are not degraded cases that
+ * return partial data; they cannot be audited, full stop, and the only useful
+ * response is to say so before spending a request finding out.
+ */
+export type ReachVerdict = "public" | "loopback" | "private" | "non_http" | "malformed";
+export interface ReachResult {
+    verdict: ReachVerdict;
+    auditable: boolean;
+    reason?: string;
+}
+export declare function checkPublicReachability(rawUrl: string): ReachResult;
+export declare function isSessionGated(rawUrl: string): boolean;

package/dist/utils/publicUrl.js

@@ -0,0 +1,115 @@
+/**
+ * Whether PSI can reach a URL at all.
+ *
+ * PSI fetches the target from Google's own infrastructure, so "can I open this
+ * in my browser" is not the test — the page has to be reachable from the
+ * public internet. Localhost and RFC1918 addresses are not degraded cases that
+ * return partial data; they cannot be audited, full stop, and the only useful
+ * response is to say so before spending a request finding out.
+ */
+/** IPv4 in a private or link-local range, per RFC1918 / RFC3927. */
+function isPrivateIPv4(host) {
+    const parts = host.split(".");
+    if (parts.length !== 4)
+        return false;
+    const [a, b] = parts.map(Number);
+    if (parts.some((p) => !/^\d+$/.test(p)) || [a, b].some(Number.isNaN))
+        return false;
+    if (a === 10)
+        return true;
+    if (a === 172 && b >= 16 && b <= 31)
+        return true;
+    if (a === 192 && b === 168)
+        return true;
+    if (a === 169 && b === 254)
+        return true; // link-local
+    if (a === 100 && b >= 64 && b <= 127)
+        return true; // CGNAT
+    return false;
+}
+function isLoopback(host) {
+    if (host === "localhost" || host.endsWith(".localhost"))
+        return true;
+    if (host === "::1" || host === "[::1]")
+        return true;
+    return /^127\./.test(host);
+}
+/**
+ * Hostnames that never resolve on the public internet. `.local` is mDNS,
+ * `.internal` and `.test`/`.invalid` are reserved, and a bare hostname with no
+ * dot is a LAN name.
+ */
+function isNonPublicName(host) {
+    if (/\.(local|internal|localdomain|test|invalid|example)$/.test(host))
+        return true;
+    return !host.includes(".") && !/^\d+$/.test(host);
+}
+export function checkPublicReachability(rawUrl) {
+    let url;
+    try {
+        url = new URL(rawUrl);
+    }
+    catch {
+        return { verdict: "malformed", auditable: false, reason: `"${rawUrl}" is not a valid URL.` };
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        return {
+            verdict: "non_http",
+            auditable: false,
+            reason: `PSI only audits http(s) URLs; got "${url.protocol}".`,
+        };
+    }
+    const host = url.hostname.toLowerCase();
+    if (isLoopback(host)) {
+        return {
+            verdict: "loopback",
+            auditable: false,
+            reason: "PSI fetches pages from Google's infrastructure, so it cannot reach " +
+                "localhost. Use run_lighthouse, which runs Chrome on this machine.",
+        };
+    }
+    if (isPrivateIPv4(host)) {
+        return {
+            verdict: "private",
+            auditable: false,
+            reason: `${host} is a private-network address and is not reachable from ` +
+                "Google's infrastructure. Use run_lighthouse for internal hosts.",
+        };
+    }
+    if (isNonPublicName(host)) {
+        return {
+            verdict: "private",
+            auditable: false,
+            reason: `"${host}" is not a publicly resolvable hostname. Use run_lighthouse ` +
+                "for hosts that only resolve on your network.",
+        };
+    }
+    return { verdict: "public", auditable: true };
+}
+/**
+ * Paths that PSI can fetch but cannot meaningfully audit, because an anonymous
+ * request does not see the real page.
+ *
+ * PSI has no session: it audits an empty cart, a redirect to a login form, or
+ * a search page with no query, and reports the result as if it were the page
+ * someone asked about. A confidently wrong number is worse than a gap, so
+ * these are flagged rather than run. `run_lighthouse` can carry cookies and is
+ * the right tool for them.
+ */
+const SESSION_GATED = [
+    /(^|\/)(cart|basket|bag)(\/|$)/,
+    /(^|\/)(checkout|payment|order-confirmation)(\/|$)/,
+    /(^|\/)(account|my-account|profile|dashboard|orders)(\/|$)/,
+    /(^|\/)(login|signin|sign-in|register|signup|sign-up|logout)(\/|$)/,
+    /(^|\/)(wishlist|favorites|favourites|saved)(\/|$)/,
+];
+export function isSessionGated(rawUrl) {
+    try {
+        const path = new URL(rawUrl).pathname.toLowerCase();
+        return SESSION_GATED.some((re) => re.test(path));
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=publicUrl.js.map

package/dist/utils/publicUrl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"publicUrl.js","sourceRoot":"","sources":["../../src/utils/publicUrl.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAUH,oEAAoE;AACpE,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnF,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,aAAa;IACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,QAAQ;IAC3D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IACpD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnF,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAc;IACpD,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,MAAM,uBAAuB,EAAE,CAAC;IAC/F,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,SAAS,EAAE,KAAK;YAChB,MAAM,EAAE,sCAAsC,GAAG,CAAC,QAAQ,IAAI;SAC/D,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAExC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,SAAS,EAAE,KAAK;YAChB,MAAM,EACJ,qEAAqE;gBACrE,mEAAmE;SACtE,CAAC;IACJ,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,KAAK;YAChB,MAAM,EACJ,GAAG,IAAI,0DAA0D;gBACjE,iEAAiE;SACpE,CAAC;IACJ,CAAC;IACD,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,KAAK;YAChB,MAAM,EACJ,IAAI,IAAI,8DAA8D;gBACtE,8CAA8C;SACjD,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,aAAa,GAAG;IACpB,+BAA+B;IAC/B,mDAAmD;IACnD,2DAA2D;IAC3D,mEAAmE;IACnE,mDAAmD;CACpD,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACpD,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/utils/sitemapReader.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Sitemap discovery.
+ *
+ * The cheapest honest way to learn which pages a site has. A sitemap is
+ * authoritative (the site published it), instant (one or two requests), and
+ * needs no crawling, no robots.txt politeness budget and no HTML parsing. A
+ * crawler is the fallback for sites without one, and is deliberately not built
+ * yet — everything downstream works without it.
+ *
+ * Sitemaps are parsed with regular expressions rather than an XML library.
+ * That is normally a mistake, but a sitemap is a machine-generated document
+ * with a fixed two-element vocabulary, and the alternative is a dependency
+ * carried by every install of this server for one code path. The parser reads
+ * <loc> only and ignores everything else, so malformed markup degrades to
+ * fewer URLs rather than to wrong ones.
+ */
+export interface SitemapResult {
+    urls: string[];
+    /** Every sitemap document actually fetched, in order. */
+    sitemapsRead: string[];
+    /** True when a cap stopped the walk early — the URL list is a subset. */
+    truncated: boolean;
+    /** Every location tried, so a failure can say what was ruled out. */
+    attempted: string[];
+    warnings: string[];
+}
+export declare function readSitemap(origin: string): Promise<SitemapResult>;

package/dist/utils/sitemapReader.js

@@ -0,0 +1,272 @@
+/**
+ * Sitemap discovery.
+ *
+ * The cheapest honest way to learn which pages a site has. A sitemap is
+ * authoritative (the site published it), instant (one or two requests), and
+ * needs no crawling, no robots.txt politeness budget and no HTML parsing. A
+ * crawler is the fallback for sites without one, and is deliberately not built
+ * yet — everything downstream works without it.
+ *
+ * Sitemaps are parsed with regular expressions rather than an XML library.
+ * That is normally a mistake, but a sitemap is a machine-generated document
+ * with a fixed two-element vocabulary, and the alternative is a dependency
+ * carried by every install of this server for one code path. The parser reads
+ * <loc> only and ignores everything else, so malformed markup degrades to
+ * fewer URLs rather than to wrong ones.
+ */
+import { gunzipSync } from "node:zlib";
+import { httpGet } from "./httpClient.js";
+/** A sitemap index can point at hundreds of children; walking all of them is rarely worth it. */
+const MAX_SITEMAPS = 25;
+const MAX_URLS = 50_000;
+const FETCH_TIMEOUT_MS = 20_000;
+const LOC_RE = /<loc>\s*([^<\s][^<]*?)\s*<\/loc>/gi;
+function decodeXmlEntities(value) {
+    return value
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'")
+        .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d)))
+        // Ampersand last, so "&amp;lt;" does not become "<".
+        .replace(/&amp;/g, "&");
+}
+function extractLocs(xml) {
+    const out = [];
+    for (const match of xml.matchAll(LOC_RE)) {
+        const loc = decodeXmlEntities(match[1].trim());
+        if (loc)
+            out.push(loc);
+    }
+    return out;
+}
+/** A <sitemapindex> points at more sitemaps; a <urlset> holds pages. */
+function isSitemapIndex(xml) {
+    return /<sitemapindex[\s>]/i.test(xml);
+}
+/**
+ * Sitemap locations declared in robots.txt.
+ *
+ * Large sites frequently do not serve /sitemap.xml and only announce the real
+ * location here, so checking robots first turns a "no sitemap found" dead end
+ * into a hit. It is also the polite thing to read before touching a site.
+ */
+async function sitemapsFromRobots(origin) {
+    const robotsUrl = new URL("/robots.txt", origin).toString();
+    const result = await httpGet(robotsUrl, { timeoutMs: 10_000, retries: 1 });
+    if (!result.ok)
+        return [];
+    const found = [];
+    for (const line of result.body.split(/\r?\n/)) {
+        const match = /^\s*sitemap:\s*(\S+)/i.exec(line);
+        if (match)
+            found.push(match[1]);
+    }
+    return found;
+}
+/**
+ * Non-HTML entries a performance audit should never spend a PSI call on.
+ * Image and video sitemaps are common and would otherwise fill the sample.
+ */
+function looksAuditable(url) {
+    return !/\.(jpe?g|png|gif|webp|avif|svg|ico|pdf|zip|gz|mp4|webm|mp3|xml|json|txt|css|js)(\?|$)/i.test(url);
+}
+/**
+ * Sitemap locations worth guessing, beyond the two standard ones.
+ *
+ * Ordered by how often they pay off. These only run when the standard
+ * locations and robots.txt have produced nothing, so the common case still
+ * costs one or two requests — a 404 from Google's edge is cheap, but seven of
+ * them on every audit would not be.
+ */
+const CANDIDATE_PATHS = [
+    "/sitemap-index.xml",
+    "/sitemap/sitemap.xml",
+    "/sitemap/index.xml",
+    "/wp-sitemap.xml", // WordPress 5.5+
+    "/sitemap_index.xml.gz",
+    "/sitemap1.xml",
+    "/sitemap.txt", // plain text, one URL per line
+];
+/**
+ * `<link rel="sitemap">` in the homepage head.
+ *
+ * Rare but authoritative when present, and it costs one request we can often
+ * justify anyway. Only consulted after the cheaper guesses fail.
+ */
+async function sitemapFromHomepageLink(origin) {
+    const result = await httpGet(origin, { timeoutMs: 15_000, retries: 0 });
+    if (!result.ok)
+        return [];
+    const found = [];
+    for (const match of result.body.matchAll(/<link\b[^>]*>/gi)) {
+        const tag = match[0];
+        if (!/rel=["']?sitemap["']?/i.test(tag))
+            continue;
+        const href = /href=["']([^"']+)["']/i.exec(tag)?.[1];
+        if (href) {
+            try {
+                found.push(new URL(href, origin).toString());
+            }
+            catch {
+                // Unparseable href — ignore rather than fail discovery over it.
+            }
+        }
+    }
+    return found;
+}
+/**
+ * A sitemap.txt is a bare newline-delimited URL list, not XML. Detected by
+ * content rather than extension, because servers are inconsistent about both.
+ */
+function isPlainTextSitemap(body) {
+    const head = body.trimStart().slice(0, 200);
+    return !head.startsWith("<") && /^https?:\/\//im.test(head);
+}
+function extractPlainTextUrls(body) {
+    return body
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => /^https?:\/\//i.test(line));
+}
+export async function readSitemap(origin) {
+    const warnings = [];
+    const sitemapsRead = [];
+    const attempted = [];
+    const urls = new Set();
+    const seen = new Set();
+    let truncated = false;
+    /**
+     * Tiers, cheapest and most authoritative first. Each is tried only if every
+     * earlier one came up empty, so a site that declares its sitemap properly
+     * still costs two requests while a site that hides it gets a real search.
+     */
+    const declared = await sitemapsFromRobots(origin);
+    const tiers = [
+        { name: "robots.txt", locations: async () => declared },
+        {
+            name: "standard locations",
+            locations: async () => [
+                new URL("/sitemap.xml", origin).toString(),
+                new URL("/sitemap_index.xml", origin).toString(),
+            ],
+        },
+        { name: "homepage <link rel=sitemap>", locations: () => sitemapFromHomepageLink(origin) },
+        {
+            name: "common CMS locations",
+            locations: async () => CANDIDATE_PATHS.map((path) => new URL(path, origin).toString()),
+        },
+    ];
+    for (const tier of tiers) {
+        // Stop on the first tier that finds a sitemap *document*, not the first
+        // that yields URLs. web.dev serves a valid index whose children time out;
+        // falling through on an empty result sent discovery guessing at nine more
+        // locations for 43s when the sitemap had already been located.
+        if (sitemapsRead.length > 0)
+            break;
+        const queue = await tier.locations();
+        let consecutiveFailures = 0;
+        while (queue.length > 0) {
+            if (sitemapsRead.length >= MAX_SITEMAPS || urls.size >= MAX_URLS) {
+                truncated = true;
+                break;
+            }
+            const next = queue.shift();
+            if (seen.has(next))
+                continue;
+            seen.add(next);
+            attempted.push(next);
+            // Large sites commonly serve .xml.gz. It arrives as an opaque gzip payload
+            // rather than with Content-Encoding, so fetch does not inflate it and a
+            // text read produces binary noise — hence the explicit binary path.
+            const gzipped = /\.gz(\?|$)/i.test(next);
+            // No retry: discovery must stay fast, and a sitemap that times out once
+            // usually times out again. Retrying doubled the cost of the slow case
+            // for no observed benefit.
+            const result = await httpGet(next, {
+                timeoutMs: FETCH_TIMEOUT_MS,
+                retries: 0,
+                raw: gzipped,
+            });
+            if (!result.ok) {
+                // A 404 or 403 on a guessed location is the expected answer to "is it
+                // here?" — cheap, informative, and no reason to stop guessing. Only
+                // expensive failures (timeouts, 5xx) count toward giving up.
+                if (result.status === 404 || result.status === 403)
+                    continue;
+                consecutiveFailures++;
+                // A sitemap index can list hundreds of children. If the first few all
+                // time out, the rest almost certainly will too, at a full timeout each.
+                if (consecutiveFailures >= 3) {
+                    warnings.push(`Gave up on ${tier.name} after ${consecutiveFailures} consecutive failures ` +
+                        `(last: ${result.error ?? `HTTP ${result.status}`}).`);
+                    break;
+                }
+                warnings.push(`Could not read ${next}: ${result.error ?? `HTTP ${result.status}`}`);
+                continue;
+            }
+            consecutiveFailures = 0;
+            let body = result.body;
+            if (gzipped) {
+                try {
+                    body = gunzipSync(Buffer.from(result.bytes ?? new Uint8Array())).toString("utf8");
+                }
+                catch (err) {
+                    warnings.push(`Could not decompress ${next}: ${err.message}`);
+                    continue;
+                }
+            }
+            if (isPlainTextSitemap(body)) {
+                const plain = extractPlainTextUrls(body).filter(looksAuditable);
+                if (plain.length === 0)
+                    continue;
+                sitemapsRead.push(next);
+                for (const loc of plain) {
+                    if (urls.size >= MAX_URLS) {
+                        truncated = true;
+                        break;
+                    }
+                    urls.add(loc);
+                }
+                continue;
+            }
+            const locs = extractLocs(body);
+            if (locs.length === 0)
+                continue;
+            sitemapsRead.push(next);
+            if (isSitemapIndex(body)) {
+                for (const child of locs)
+                    if (!seen.has(child))
+                        queue.push(child);
+                continue;
+            }
+            for (const loc of locs) {
+                if (urls.size >= MAX_URLS) {
+                    truncated = true;
+                    break;
+                }
+                if (looksAuditable(loc))
+                    urls.add(loc);
+            }
+        }
+    }
+    if (sitemapsRead.length > 0 && urls.size === 0) {
+        warnings.push(`Found a sitemap at ${sitemapsRead[0]} but could not extract any URLs from it — ` +
+            `its child documents failed to load or contained no page entries. Supply URLs ` +
+            `with discovery:"list" or discovery:"csv".`);
+    }
+    if (sitemapsRead.length === 0) {
+        warnings.push(`No sitemap found for ${origin}. Tried ${attempted.length} location(s): ` +
+            `robots.txt, the standard paths, the homepage <link rel="sitemap">, and ` +
+            `common CMS locations. ` +
+            `Supply URLs directly with discovery:"list", or point at a CSV export with ` +
+            `discovery:"csv" — an analytics top-pages export is the better input for a ` +
+            `performance audit anyway, since it is weighted by real traffic.`);
+    }
+    if (truncated) {
+        warnings.push(`Stopped after ${sitemapsRead.length} sitemap documents and ${urls.size} URLs. ` +
+            `The template breakdown below is based on that subset.`);
+    }
+    return { urls: [...urls], sitemapsRead, truncated, attempted, warnings };
+}
+//# sourceMappingURL=sitemapReader.js.map

package/dist/utils/sitemapReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemapReader.js","sourceRoot":"","sources":["../../src/utils/sitemapReader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAa1C,iGAAiG;AACjG,MAAM,YAAY,GAAG,EAAE,CAAC;AACxB,MAAM,QAAQ,GAAG,MAAM,CAAC;AACxB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,GAAG,oCAAoC,CAAC;AAEpD,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,qDAAqD;SACpD,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wEAAwE;AACxE,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,MAAc;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,CAAC,wFAAwF,CAAC,IAAI,CACnG,GAAG,CACJ,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG;IACtB,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB;IACpB,iBAAiB,EAAS,iBAAiB;IAC3C,uBAAuB;IACvB,eAAe;IACf,cAAc,EAAY,+BAA+B;CAC1D,CAAC;AAEF;;;;;GAKG;AACH,KAAK,UAAU,uBAAuB,CAAC,MAAc;IACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAClD,MAAM,IAAI,GAAG,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,gEAAgE;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,OAAO,IAAI;SACR,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc;IAC9C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB;;;;OAIG;IACH,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,KAAK,GAAgE;QACzE,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,QAAQ,EAAE;QACvD;YACE,IAAI,EAAE,oBAAoB;YAC1B,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC;gBACrB,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE;gBAC1C,IAAI,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE;aACjD;SACF;QACD,EAAE,IAAI,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE;QACzF;YACE,IAAI,EAAE,sBAAsB;YAC5B,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;SACvF;KACF,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,wEAAwE;QACxE,0EAA0E;QAC1E,0EAA0E;QAC1E,+DAA+D;QAC/D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM;QAEnC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;gBACjE,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;YAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAY,CAAC;YACrC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAErB,2EAA2E;YAC3E,wEAAwE;YACxE,oEAAoE;YACpE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,wEAAwE;YACxE,sEAAsE;YACtE,2BAA2B;YAC3B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE;gBACjC,SAAS,EAAE,gBAAgB;gBAC3B,OAAO,EAAE,CAAC;gBACV,GAAG,EAAE,OAAO;aACb,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,sEAAsE;gBACtE,oEAAoE;gBACpE,6DAA6D;gBAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;oBAAE,SAAS;gBAE7D,mBAAmB,EAAE,CAAC;gBACtB,sEAAsE;gBACtE,wEAAwE;gBACxE,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;oBAC7B,QAAQ,CAAC,IAAI,CACX,cAAc,IAAI,CAAC,IAAI,UAAU,mBAAmB,wBAAwB;wBAC1E,UAAU,MAAM,CAAC,KAAK,IAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,IAAI,CACxD,CAAC;oBACF,MAAM;gBACR,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,kBAAkB,IAAI,KAAK,MAAM,CAAC,KAAK,IAAI,QAAQ,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,mBAAmB,GAAG,CAAC,CAAC;YAExB,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACH,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACpF,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,QAAQ,CAAC,IAAI,CAAC,wBAAwB,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;oBACzE,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBACjC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;wBAC1B,SAAS,GAAG,IAAI,CAAC;wBACjB,MAAM;oBACR,CAAC;oBACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAChC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAExB,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,KAAK,IAAI,IAAI;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;wBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClE,SAAS;YACX,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC1B,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM;gBACR,CAAC;gBACD,IAAI,cAAc,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CACX,sBAAsB,YAAY,CAAC,CAAC,CAAC,4CAA4C;YAC/E,+EAA+E;YAC/E,2CAA2C,CAC9C,CAAC;IACJ,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CACX,wBAAwB,MAAM,WAAW,SAAS,CAAC,MAAM,gBAAgB;YACvE,yEAAyE;YACzE,wBAAwB;YACxB,4EAA4E;YAC5E,4EAA4E;YAC5E,iEAAiE,CACpE,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CACX,iBAAiB,YAAY,CAAC,MAAM,0BAA0B,IAAI,CAAC,IAAI,SAAS;YAC9E,uDAAuD,CAC1D,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC3E,CAAC"}

package/dist/utils/urlClassifier.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Cluster a URL list into page templates.
+ *
+ * A performance audit's unit of analysis is the template, not the URL. Nobody
+ * needs 3,904 product pages measured; they need to know what a product page
+ * costs. The manual Five Below audit was organised exactly this way — Homepage,
+ * PLP (category), PLP (subcategory), PLP (paginated), PDP, Cart, Search,
+ * Info — with representative URLs sampled per template, and this reproduces
+ * that structure automatically instead of asking someone to hand-write the
+ * list.
+ *
+ * The clustering is a heuristic and will occasionally be wrong. That is
+ * acceptable because the plan tool shows its work and the user approves the
+ * sample before any PSI call is spent: a misclassification costs a
+ * conversation turn, not quota.
+ */
+export interface UrlTemplate {
+    id: string;
+    label: string;
+    /** Path shape with high-cardinality segments collapsed, e.g. "/categories/*". */
+    pattern: string;
+    urlCount: number;
+    /** How many URLs to audit, given the population size. */
+    suggestedSample: number;
+    /** Representative URLs, most-canonical first. */
+    candidates: string[];
+    auditable: boolean;
+    reason?: string;
+    recommendation?: string;
+    /**
+     * True when *any* URL in the template is session-gated — computed over the
+     * whole group, not the sampled candidates. Deciding this from candidates made
+     * the verdict depend on which URLs the sampler happened to pick, so a
+     * template containing /cart could pass as auditable purely by luck.
+     */
+    anySessionGated: boolean;
+}
+export interface ClassifyOptions {
+    /** Patterns matching fewer URLs than this are folded into "Other". */
+    minTemplateSize?: number;
+}
+export declare function classifyUrls(rawUrls: string[], options?: ClassifyOptions): {
+    templates: UrlTemplate[];
+    skipped: number;
+};