@rivi-software/agentic-seo-check 0.1.1 → 0.3.0

package/dist/core/rules/urls.js

@@ -0,0 +1,293 @@
+import { hostOf, resolveUrl } from "../url.js";
+import { isLocalHost } from "./canonical.js";
+import { cap, defineRule, GOOGLE_DOCS } from "./rule.js";
+/** Query parameters that only tell analytics where a click came from. */
+const TRACKING_PARAMETERS = new Set([
+    "gclid",
+    "gbraid",
+    "wbraid",
+    "dclid",
+    "fbclid",
+    "msclkid",
+    "yclid",
+    "twclid",
+    "ttclid",
+    "igshid",
+    "li_fat_id",
+    "mc_cid",
+    "mc_eid",
+    "_ga",
+    "_gl",
+    "_hsenc",
+    "_hsmi",
+]);
+const isTrackingParameter = (name) => {
+    const key = name.toLowerCase();
+    return key.startsWith("utm_") || TRACKING_PARAMETERS.has(key);
+};
+/** Parameter names servers use for session ids, and the shortest value that looks like one. */
+const SESSION_PARAMETERS = new Map([
+    ["jsessionid", 8],
+    ["phpsessid", 8],
+    ["sessionid", 8],
+    ["session_id", 8],
+    ["sessid", 8],
+    ["cfid", 4],
+    ["cftoken", 8],
+    ["zenid", 8],
+    ["oscsid", 8],
+    // "sid" also stands for a site or store id; only a long token is a session.
+    ["sid", 16],
+]);
+/** A session id in the URL's query or as a `;jsessionid=` path parameter, or null. */
+const sessionParameter = (url) => {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    if (/;jsessionid=/iu.test(parsed.pathname)) {
+        return "jsessionid";
+    }
+    for (const [name, value] of parsed.searchParams) {
+        const key = name.toLowerCase();
+        const minLength = key.startsWith("aspsessionid")
+            ? 8
+            : SESSION_PARAMETERS.get(key);
+        if (minLength !== undefined && value.length >= minLength) {
+            return name;
+        }
+    }
+    return null;
+};
+/** The page URL's path, or null when the URL is not one a visitor can request. */
+const pagePath = (page, source) => {
+    if (source === "html") {
+        return null;
+    }
+    try {
+        return new URL(page.url).pathname;
+    }
+    catch {
+        return null;
+    }
+};
+/**
+ * A locale segment such as en-US, en-us, pt_BR, es-419, zh-Hans or zh-Hant-TW, whose case and
+ * underscore are conventions: a lowercase language, then an optional Title-case script and an
+ * optional two-letter or three-digit region. "Our-Team" and "how_to_cook" do not match.
+ */
+const LOCALE_SEGMENT = /^[a-z]{2,3}(?:[-_][A-Z][a-z]{3})?(?:[-_](?:[A-Za-z]{2}|\d{3}))?$/u;
+/** The path without locale segments, which are spelled the way the locale is. */
+const pathWithoutLocales = (path) => path
+    .split("/")
+    .filter((segment) => !LOCALE_SEGMENT.test(segment))
+    .join("/");
+const KEEP_EXISTING_URLS = "Do not rename URLs that are already indexed or linked to only for this: a rename needs a 301 redirect from every old URL and resets what search engines know about the page. Apply it to new URLs.";
+/** Private networks: an address there resolves only inside one office or one machine. */
+const isPrivateAddress = (host) => /^(?:10|127)\.\d{1,3}\.\d{1,3}\.\d{1,3}$/u.test(host) ||
+    /^192\.168\.\d{1,3}\.\d{1,3}$/u.test(host) ||
+    /^172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}$/u.test(host);
+const isDevHost = (url) => {
+    const host = hostOf(url);
+    return host !== null && (isLocalHost(url) || isPrivateAddress(host));
+};
+/** Every URL the page links to or loads from, once, with the first place it is written. */
+const urlsOnPage = (page) => {
+    const out = new Map();
+    const add = (raw, where) => {
+        const url = raw ? resolveUrl(raw, page.url) : null;
+        if (url && !out.has(url)) {
+            out.set(url, where);
+        }
+    };
+    add(page.canonical, '<link rel="canonical">');
+    for (const h of page.hreflang) {
+        add(h.url, `<link rel="alternate" hreflang="${h.lang}">`);
+    }
+    add(page.og["url"], "og:url");
+    add(page.og["image"], "og:image");
+    for (const link of page.links) {
+        add(link.url, "<a href>");
+    }
+    for (const image of page.images) {
+        add(image.src, "<img src>");
+    }
+    for (const resource of page.resources) {
+        add(resource.url, resource.kind);
+    }
+    return [...out].map(([url, where]) => ({ url, where }));
+};
+export const urlRules = [
+    defineRule({
+        id: "url.uppercase",
+        category: "links",
+        severity: "info",
+        description: "The page's URL path has uppercase letters. Google treats /Page and /page as two URLs, so links that differ in case split signals or end in a 404.",
+        recommendation: `Use lowercase paths for new pages, and redirect (301) other spellings of the path to the one you keep. ${KEEP_EXISTING_URLS}`,
+        docs: GOOGLE_DOCS.urlStructure,
+    }, ({ page, source }, make) => {
+        const path = pagePath(page, source);
+        // Percent-escapes such as %C3%A9 are uppercase by convention, not part of the path's spelling.
+        return path &&
+            /[A-Z]/u.test(pathWithoutLocales(path).replaceAll(/%[\dA-F]{2}/giu, ""))
+            ? make(`The URL path ${path} has uppercase letters.`, {
+                evidence: { path },
+            })
+            : null;
+    }),
+    defineRule({
+        id: "url.underscores",
+        category: "links",
+        severity: "info",
+        description: "The page's URL path separates words with underscores; Google recommends hyphens.",
+        recommendation: `Separate words in new URLs with hyphens (/blue-widgets, not /blue_widgets). ${KEEP_EXISTING_URLS}`,
+        docs: GOOGLE_DOCS.urlStructure,
+    }, ({ page, source }, make) => {
+        const path = pagePath(page, source);
+        // Underscores in a word (/blue_widgets), not as a leading marker (/_next/, /_assets/).
+        return path && /[\p{L}\p{N}]_[\p{L}\p{N}]/u.test(pathWithoutLocales(path))
+            ? make(`The URL path ${path} uses underscores between words.`, {
+                evidence: { path },
+            })
+            : null;
+    }),
+    defineRule({
+        id: "url.session-id",
+        category: "links",
+        severity: "warning",
+        description: "The page's URL or its internal links carry a session id (jsessionid, PHPSESSID, sid), so every visit creates new URLs for the same content.",
+        recommendation: "Keep sessions in a cookie, not in the URL. In Java servlet containers, set the session tracking mode to COOKIE; in PHP, set session.use_trans_sid=0 and session.use_only_cookies=1.",
+        docs: GOOGLE_DOCS.urlStructure,
+    }, ({ page, source }, make) => {
+        const own = pagePath(page, source) ? sessionParameter(page.url) : null;
+        const links = page.links.filter((l) => l.internal && l.url && sessionParameter(l.url));
+        if (!own && links.length === 0) {
+            return null;
+        }
+        const parameter = own ?? (links[0]?.url ? sessionParameter(links[0].url) : null);
+        return make(own
+            ? `The page URL carries a session id (${own}).`
+            : `${links.length} internal link(s) carry a session id (${parameter}).`, {
+            evidence: {
+                parameter,
+                pageUrl: own ? page.url : null,
+                links: links.length,
+                examples: cap(links.map((l) => l.url)),
+            },
+        });
+    }),
+    defineRule({
+        id: "links.tracking-parameters",
+        category: "links",
+        severity: "info",
+        description: "Internal links carry campaign tracking parameters (utm_*, gclid, fbclid), which make crawlers fetch duplicate URLs and overwrite the visit's real traffic source in analytics.",
+        recommendation: "Link to the clean URL. Campaign parameters belong on links from emails, ads and other sites, not on links between your own pages; to measure which banner or menu item was clicked, use an analytics event instead.",
+        docs: GOOGLE_DOCS.urlStructure,
+    }, ({ page }, make) => {
+        const tagged = page.links.filter((l) => {
+            if (!l.internal || !l.url) {
+                return false;
+            }
+            try {
+                return [...new URL(l.url).searchParams.keys()].some(isTrackingParameter);
+            }
+            catch {
+                return false;
+            }
+        });
+        return tagged.length > 0
+            ? make(`${tagged.length} internal link(s) carry tracking parameters.`, {
+                evidence: {
+                    count: tagged.length,
+                    examples: cap(tagged.map((l) => l.url)),
+                },
+            })
+            : null;
+    }),
+    defineRule({
+        id: "links.local-host",
+        category: "links",
+        severity: "error",
+        description: "A public page's canonical, og:url, og:image, hreflang, images or scripts point to localhost, a .local or .test host, or a private network address, which work only on the developer's machine (error). Only <a> links there is info: a setup guide may mean them.",
+        recommendation: "Build these URLs from the production site URL (an environment variable such as SITE_URL) instead of the development server's, or make them relative. Check the canonical, og:url and og:image first: a canonical on localhost points search engines at a page they cannot reach.",
+    }, ({ page, source }, make) => {
+        if (source === "html" || isDevHost(page.url)) {
+            return null;
+        }
+        const local = urlsOnPage(page).filter((u) => isDevHost(u.url));
+        if (local.length === 0) {
+            return null;
+        }
+        const places = [...new Set(local.map((u) => u.where))];
+        // A tutorial may link to http://localhost:3000 on purpose, for the reader's own machine.
+        const onlyAnchors = places.every((w) => w === "<a href>");
+        return make(`${local.length} URL(s) on the page point to a development host (${cap(places, 4).join(", ")}).`, {
+            severity: onlyAnchors ? "info" : "error",
+            recommendation: onlyAnchors
+                ? "If these links are meant for the reader's own machine (a setup guide), leave them. Otherwise build them from the production site URL or make them relative."
+                : undefined,
+            evidence: {
+                count: local.length,
+                examples: cap(local.map((u) => `${u.where}: ${u.url}`)),
+            },
+        });
+    }),
+    defineRule({
+        id: "links.missing-fragment",
+        category: "links",
+        severity: "info",
+        description: 'Same-page links (href="#name") point to an id that no element on the page has, so the link goes nowhere.',
+        recommendation: 'Give the target element the id the link names, or fix the link. If JavaScript handles the click (a menu or modal toggle), use a <button> instead of <a href="#...">.',
+    }, ({ page }, make) => page.missingFragments.length > 0
+        ? make(`${page.missingFragments.length} same-page link(s) point to ids that are not on the page: ${cap(page.missingFragments, 5).join(", ")}.`, {
+            selector: cap(page.missingFragments, 5)
+                .map((href) => `a[href="${href.replaceAll('"', '\\"')}"]`)
+                .join(", "),
+            evidence: { fragments: cap(page.missingFragments) },
+        })
+        : null),
+    defineRule({
+        id: "https.mixed-content",
+        category: "document",
+        severity: "warning",
+        description: "An https page loads resources over http. Browsers block scripts, stylesheets, frames and responsive images (warning) and upgrade plain images, audio and video to https, which fails when the https URL does not exist (info). With a Content-Security-Policy of upgrade-insecure-requests, every resource is upgraded (info).",
+        recommendation: "Load every resource over https: change http:// URLs to https:// (or make them relative), and host or replace any resource that has no https version. A Content-Security-Policy of upgrade-insecure-requests upgrades the rest.",
+        docs: "https://web.dev/articles/what-is-mixed-content",
+    }, ({ page, source, response }, make) => {
+        if (source !== "url" || !page.url.startsWith("https://")) {
+            return null;
+        }
+        const insecure = page.resources.filter((r) => r.url.startsWith("http://"));
+        // upgrade-insecure-requests (meta tag or header) has the browser request all of them over https.
+        const upgradesAll = page.upgradesInsecureRequests ||
+            // Several CSP headers arrive joined with ", ", and a comma separates policies.
+            /(?:^|[;,])\s*upgrade-insecure-requests\s*(?:[;,]|$)/iu.test(response?.headers["content-security-policy"] ?? "");
+        const isUpgraded = (r) => upgradesAll || r.kind === "image" || r.kind === "media";
+        const blocked = insecure.filter((r) => !isUpgraded(r));
+        const upgraded = insecure.filter(isUpgraded);
+        const examples = (list) => cap(list.map((r) => `${r.kind}: ${r.url}`));
+        return [
+            blocked.length > 0
+                ? make(`Browsers block ${blocked.length} resource(s) the page loads over http (${[...new Set(blocked.map((r) => r.kind))].join(", ")}).`, {
+                    evidence: {
+                        count: blocked.length,
+                        examples: examples(blocked),
+                    },
+                })
+                : null,
+            upgraded.length > 0
+                ? make(`${upgraded.length} resource URL(s) use http; browsers upgrade them to https, which fails where the https URL does not exist.`, {
+                    severity: "info",
+                    evidence: {
+                        count: upgraded.length,
+                        examples: examples(upgraded),
+                    },
+                })
+                : null,
+        ].filter((f) => f !== null);
+    }),
+];
+//# sourceMappingURL=urls.js.map

package/dist/core/rules/urls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"urls.js","sourceRoot":"","sources":["../../../src/core/rules/urls.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAEzD,yEAAyE;AACzE,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,SAAS;IACT,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,KAAK;IACL,QAAQ;IACR,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG,CAAC,IAAY,EAAW,EAAE;IACpD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAE/B,OAAO,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,+FAA+F;AAC/F,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,CAAC,YAAY,EAAE,CAAC,CAAC;IACjB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,YAAY,EAAE,CAAC,CAAC;IACjB,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,OAAO,EAAE,CAAC,CAAC;IACZ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,4EAA4E;IAC5E,CAAC,KAAK,EAAE,EAAE,CAAC;CACZ,CAAC,CAAC;AAEH,sFAAsF;AACtF,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAiB,EAAE;IACtD,IAAI,MAAW,CAAC;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAE/B,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC;YAC9C,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEhC,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,IAAmB,EAAE,MAAkB,EAAiB,EAAE;IAC1E,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,cAAc,GAClB,mEAAmE,CAAC;AAEtE,iFAAiF;AACjF,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAU,EAAE,CAClD,IAAI;KACD,KAAK,CAAC,GAAG,CAAC;KACV,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KAClD,IAAI,CAAC,GAAG,CAAC,CAAC;AAEf,MAAM,kBAAkB,GACtB,oMAAoM,CAAC;AAEvM,yFAAyF;AACzF,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAW,EAAE,CACjD,0CAA0C,CAAC,IAAI,CAAC,IAAI,CAAC;IACrD,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;IAC1C,gDAAgD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAE9D,MAAM,SAAS,GAAG,CAAC,GAAW,EAAW,EAAE;IACzC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAEzB,OAAO,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,MAAM,UAAU,GAAG,CAAC,IAAmB,EAAoC,EAAE;IAC3E,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEtC,MAAM,GAAG,GAAG,CAAC,GAA8B,EAAE,KAAa,EAAE,EAAE;QAC5D,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnD,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;IAEF,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAE9C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,mCAAmC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AAC1D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,UAAU,CACR;QACE,EAAE,EAAE,eAAe;QACnB,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,MAAM;QAChB,WAAW,EACT,mJAAmJ;QACrJ,cAAc,EAAE,0GAA0G,kBAAkB,EAAE;QAC9I,IAAI,EAAE,WAAW,CAAC,YAAY;KAC/B,EACD,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAEpC,+FAA+F;QAC/F,OAAO,IAAI;YACT,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,gBAAgB,IAAI,yBAAyB,EAAE;gBAClD,QAAQ,EAAE,EAAE,IAAI,EAAE;aACnB,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;IACX,CAAC,CACF;IACD,UAAU,CACR;QACE,EAAE,EAAE,iBAAiB;QACrB,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,MAAM;QAChB,WAAW,EACT,kFAAkF;QACpF,cAAc,EAAE,+EAA+E,kBAAkB,EAAE;QACnH,IAAI,EAAE,WAAW,CAAC,YAAY;KAC/B,EACD,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAEpC,uFAAuF;QACvF,OAAO,IAAI,IAAI,4BAA4B,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,gBAAgB,IAAI,kCAAkC,EAAE;gBAC3D,QAAQ,EAAE,EAAE,IAAI,EAAE;aACnB,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;IACX,CAAC,CACF;IACD,UAAU,CACR;QACE,EAAE,EAAE,gBAAgB;QACpB,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,SAAS;QACnB,WAAW,EACT,6IAA6I;QAC/I,cAAc,EACZ,qLAAqL;QACvL,IAAI,EAAE,WAAW,CAAC,YAAY;KAC/B,EACD,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE;QACzB,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEvE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CACtD,CAAC;QAEF,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GACb,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEjE,OAAO,IAAI,CACT,GAAG;YACD,CAAC,CAAC,sCAAsC,GAAG,IAAI;YAC/C,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,yCAAyC,SAAS,IAAI,EACzE;YACE,QAAQ,EAAE;gBACR,SAAS;gBACT,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;gBAC9B,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;aACvC;SACF,CACF,CAAC;IACJ,CAAC,CACF;IACD,UAAU,CACR;QACE,EAAE,EAAE,2BAA2B;QAC/B,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,MAAM;QAChB,WAAW,EACT,gLAAgL;QAClL,cAAc,EACZ,qNAAqN;QACvN,IAAI,EAAE,WAAW,CAAC,YAAY;KAC/B,EACD,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE;QACjB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;gBAC1B,OAAO,KAAK,CAAC;YACf,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CACjD,mBAAmB,CACpB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,8CAA8C,EAAE;gBACnE,QAAQ,EAAE;oBACR,KAAK,EAAE,MAAM,CAAC,MAAM;oBACpB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;iBACxC;aACF,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;IACX,CAAC,CACF;IACD,UAAU,CACR;QACE,EAAE,EAAE,kBAAkB;QACtB,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,OAAO;QACjB,WAAW,EACT,mQAAmQ;QACrQ,cAAc,EACZ,kRAAkR;KACrR,EACD,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE;QACzB,IAAI,MAAM,KAAK,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAE/D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvD,yFAAyF;QACzF,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC;QAE1D,OAAO,IAAI,CACT,GAAG,KAAK,CAAC,MAAM,oDAAoD,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAChG;YACE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YACxC,cAAc,EAAE,WAAW;gBACzB,CAAC,CAAC,6JAA6J;gBAC/J,CAAC,CAAC,SAAS;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;aACxD;SACF,CACF,CAAC;IACJ,CAAC,CACF;IACD,UAAU,CACR;QACE,EAAE,EAAE,wBAAwB;QAC5B,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,MAAM;QAChB,WAAW,EACT,0GAA0G;QAC5G,cAAc,EACZ,sKAAsK;KACzK,EACD,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,CACjB,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC9B,CAAC,CAAC,IAAI,CACF,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,6DAA6D,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EACvI;YACE,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;iBACpC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC;iBACzD,IAAI,CAAC,IAAI,CAAC;YACb,QAAQ,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE;SACpD,CACF;QACH,CAAC,CAAC,IAAI,CACX;IACD,UAAU,CACR;QACE,EAAE,EAAE,qBAAqB;QACzB,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,SAAS;QACnB,WAAW,EACT,gUAAgU;QAClU,cAAc,EACZ,gOAAgO;QAClO,IAAI,EAAE,gDAAgD;KACvD,EACD,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE;QACnC,IAAI,MAAM,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3C,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAC5B,CAAC;QAEF,iGAAiG;QACjG,MAAM,WAAW,GACf,IAAI,CAAC,wBAAwB;YAC7B,+EAA+E;YAC/E,uDAAuD,CAAC,IAAI,CAC1D,QAAQ,EAAE,OAAO,CAAC,yBAAyB,CAAC,IAAI,EAAE,CACnD,CAAC;QAEJ,MAAM,UAAU,GAAG,CAAC,CAA4B,EAAE,EAAE,CAClD,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;QAE1D,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAE7C,MAAM,QAAQ,GAAG,CAAC,IAAoB,EAAE,EAAE,CACxC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAE9C,OAAO;YACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,IAAI,CACF,kBAAkB,OAAO,CAAC,MAAM,0CAA0C,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EACjI;oBACE,QAAQ,EAAE;wBACR,KAAK,EAAE,OAAO,CAAC,MAAM;wBACrB,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;qBAC5B;iBACF,CACF;gBACH,CAAC,CAAC,IAAI;YACR,QAAQ,CAAC,MAAM,GAAG,CAAC;gBACjB,CAAC,CAAC,IAAI,CACF,GAAG,QAAQ,CAAC,MAAM,4GAA4G,EAC9H;oBACE,QAAQ,EAAE,MAAM;oBAChB,QAAQ,EAAE;wBACR,KAAK,EAAE,QAAQ,CAAC,MAAM;wBACtB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;qBAC7B;iBACF,CACF;gBACH,CAAC,CAAC,IAAI;SACT,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC9B,CAAC,CACF;CACF,CAAC"}

package/dist/core/score.d.ts

@@ -1,3 +1,5 @@
-import type { Finding, PageAudit, SeverityCounts } from "./types.js";
+import type { Finding, SeverityCounts } from "./types.js";
 export declare const pageScore: (counts: SeverityCounts) => number;
-export declare const siteScore: (pages: PageAudit[], siteFindings: Finding[]) => number;
+export declare const siteScore: (pages: {
+    score: number;
+}[], siteFindings: Finding[]) => number;

package/dist/core/score.js.map

@@ -1 +1 @@
-{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/core/score.ts"],"names":[],"mappings":"AAEA,MAAM,YAAY,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAW,CAAC;AACjE,MAAM,YAAY,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAW,CAAC;AAEnE,MAAM,KAAK,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;AAEnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,MAAsB,EAAU,EAAE;IAC1D,MAAM,SAAS,GACb,MAAM,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK;QACjC,MAAM,CAAC,OAAO,GAAG,YAAY,CAAC,OAAO;QACrC,MAAM,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC;IAElC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,KAAkB,EAClB,YAAuB,EACf,EAAE;IACV,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM;QACtB,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM;QAC3D,CAAC,CAAC,GAAG,CAAC;IAER,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CACnC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC1C,CAAC,CACF,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC"}
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/core/score.ts"],"names":[],"mappings":"AAEA,MAAM,YAAY,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAW,CAAC;AACjE,MAAM,YAAY,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAW,CAAC;AAEnE,MAAM,KAAK,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;AAEnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,MAAsB,EAAU,EAAE;IAC1D,MAAM,SAAS,GACb,MAAM,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK;QACjC,MAAM,CAAC,OAAO,GAAG,YAAY,CAAC,OAAO;QACrC,MAAM,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC;IAElC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,KAA0B,EAC1B,YAAuB,EACf,EAAE;IACV,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM;QACtB,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM;QAC3D,CAAC,CAAC,GAAG,CAAC;IAER,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CACnC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC1C,CAAC,CACF,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC"}

package/dist/core/site-probes.d.ts

@@ -0,0 +1,33 @@
+import type { FetchOptions } from "./fetch.js";
+import type { LinkCheck, PageRecord, UrlVariantProbe } from "./site-rules.js";
+export interface ProbeOptions {
+    fetch: FetchOptions;
+    concurrency: number;
+    delayMs: number;
+    /** robots.txt lets the crawler fetch the URL. */
+    allows: (url: string) => boolean;
+    /** The crawl's start URL: images on its site are checked, and only its origin gets `fetch.headers`. */
+    startUrl: string;
+    includeSubdomains: boolean;
+    /** Also check images on other hosts, as `checkExternalLinks` does for links. */
+    checkExternal: boolean;
+}
+/** Pages with fewer words in their HTML (an app shell) give no text to compare. */
+export declare const MIN_COMPARABLE_WORDS = 50;
+export interface ImageCheckResult {
+    imageChecks: Map<string, LinkCheck>;
+    /** For every image URL, the pages that show it. */
+    imageSources: Map<string, Set<string>>;
+}
+/** Image URLs the crawled pages show, with the pages showing each, in crawl order. */
+export declare const imageSourcesOf: (records: PageRecord[]) => Map<string, Set<string>>;
+/** Request up to `limit` image URLs, the ones on most pages first, without reading their bodies. */
+export declare const checkImages: (records: PageRecord[], limit: number, options: ProbeOptions) => Promise<ImageCheckResult>;
+/** Shared shingles from which two pages count as the same text, as in site.duplicate-content. */
+export declare const SAME_TEXT = 0.9;
+/**
+ * Request a trailing-slash and an uppercase variant of a few crawled pages. A variant that answers
+ * 200 itself, without a redirect, and names no canonical that points back is a second URL for the
+ * same page. Local dev servers often answer both forms where production does not, so they are left out.
+ */
+export declare const probeUrlVariants: (records: PageRecord[], startUrl: string, options: ProbeOptions) => Promise<UrlVariantProbe[]>;

package/dist/core/site-probes.js

@@ -0,0 +1,144 @@
+import { mapConcurrent, sleep } from "./concurrency.js";
+import { fetchPage, isHtmlResponse, probeUrl } from "./fetch.js";
+import { extractPage } from "./html/extract.js";
+import { jaccard, shingles } from "./html/text.js";
+import { isLocalHost } from "./rules/canonical.js";
+import { robotsDirectives } from "./rules/indexing.js";
+import { hostOf, isSameSite, looksLikeHtml, normalizeUrl } from "./url.js";
+/** Pages with fewer words in their HTML (an app shell) give no text to compare. */
+export const MIN_COMPARABLE_WORDS = 50;
+/** Image URLs the crawled pages show, with the pages showing each, in crawl order. */
+export const imageSourcesOf = (records) => {
+    const sources = new Map();
+    // The resources were resolved against the page's <base href>, as the browser resolves them.
+    for (const record of records) {
+        for (const resource of record.page.resources) {
+            if (resource.kind !== "image" && resource.kind !== "responsive-image") {
+                continue;
+            }
+            const set = sources.get(resource.url) ?? new Set();
+            set.add(record.url);
+            sources.set(resource.url, set);
+        }
+    }
+    return sources;
+};
+/** Request up to `limit` image URLs, the ones on most pages first, without reading their bodies. */
+export const checkImages = async (records, limit, options) => {
+    const imageSources = imageSourcesOf(records);
+    const imageChecks = new Map();
+    const startHost = hostOf(options.startUrl) ?? "";
+    const startOrigin = new URL(options.startUrl).origin;
+    // A page can point an <img> anywhere; the crawl requests other hosts only when asked to.
+    const targets = [...imageSources]
+        .toSorted((a, b) => b[1].size - a[1].size)
+        .flatMap(([url]) => options.allows(url) &&
+        (options.checkExternal ||
+            isSameSite(url, startHost, options.includeSubdomains))
+        ? [url]
+        : [])
+        .slice(0, limit);
+    await mapConcurrent(targets, options.concurrency, async (url) => {
+        // The user's headers (a staging password, a cookie) go to the audited origin only.
+        const { headers, ...rest } = options.fetch;
+        const fetchOptions = {
+            ...rest,
+            timeoutMs: Math.min(options.fetch.timeoutMs ?? 15_000, 10_000),
+        };
+        if (headers && new URL(url).origin === startOrigin) {
+            fetchOptions.headers = headers;
+        }
+        const res = await probeUrl(url, fetchOptions);
+        imageChecks.set(url, {
+            url,
+            status: res.status,
+            finalUrl: res.finalUrl,
+            redirectChain: res.redirectChain,
+            error: res.error,
+        });
+        if (options.delayMs) {
+            await sleep(options.delayMs);
+        }
+    });
+    return { imageChecks, imageSources };
+};
+/** Shared shingles from which two pages count as the same text, as in site.duplicate-content. */
+export const SAME_TEXT = 0.9;
+/** Pages whose slash and case variants are requested; a few tell how the server treats them all. */
+const VARIANT_SAMPLE = 3;
+/** The same URL with the trailing slash added or removed, and with the path in uppercase. */
+const variantsOf = (url) => {
+    const parsed = new URL(url);
+    const { pathname } = parsed;
+    const out = [];
+    const slash = new URL(parsed.href);
+    slash.pathname = pathname.endsWith("/")
+        ? pathname.slice(0, -1)
+        : `${pathname}/`;
+    out.push({ kind: "trailing-slash", variant: slash.href });
+    // Percent-escapes keep their case; only letters in the path change.
+    const upper = pathname.replaceAll(/%[\dA-F]{2}|[a-z]+/giu, (part) => part.startsWith("%") ? part : part.toUpperCase());
+    if (upper !== pathname) {
+        const cased = new URL(parsed.href);
+        cased.pathname = upper;
+        out.push({ kind: "case", variant: cased.href });
+    }
+    return out;
+};
+const isIndexablePage = (record) => record.response?.status === 200 &&
+    !robotsDirectives(record.page, record.response).noindex;
+/**
+ * Request a trailing-slash and an uppercase variant of a few crawled pages. A variant that answers
+ * 200 itself, without a redirect, and names no canonical that points back is a second URL for the
+ * same page. Local dev servers often answer both forms where production does not, so they are left out.
+ */
+export const probeUrlVariants = async (records, startUrl, options) => {
+    if (isLocalHost(startUrl)) {
+        return [];
+    }
+    const sample = records
+        .filter((r) => {
+        const { pathname } = new URL(r.url);
+        return (pathname !== "/" &&
+            r.page.wordCount >= MIN_COMPARABLE_WORDS &&
+            isIndexablePage(r) &&
+            looksLikeHtml(r.url) &&
+            !/\.[a-z\d]{2,5}$/iu.test(pathname));
+    })
+        .slice(0, VARIANT_SAMPLE);
+    const requests = sample.flatMap((record) => variantsOf(record.url)
+        .filter(({ variant }) => options.allows(variant))
+        .map((v) => ({ record, ...v })));
+    const probes = [];
+    await mapConcurrent(requests, options.concurrency, async ({ record, kind, variant }) => {
+        const res = await fetchPage(variant, options.fetch);
+        const answered = res.status === 200 &&
+            res.redirectChain.length === 0 &&
+            res.body !== null &&
+            isHtmlResponse(res);
+        const variantPage = answered
+            ? extractPage(res.body ?? "", { url: variant })
+            : null;
+        const canonical = variantPage?.canonical ?? null;
+        // The page's own canonical, or the page itself, is where both URLs should consolidate.
+        const target = normalizeUrl(record.page.canonical ?? record.url);
+        const named = normalizeUrl(canonical ?? variant);
+        // An app shell or catch-all page answering every unknown path is a soft 404, not this page.
+        const samePage = variantPage !== null &&
+            jaccard(shingles(record.page.text), shingles(variantPage.text)) >=
+                SAME_TEXT;
+        probes.push({
+            kind,
+            url: record.url,
+            variant,
+            status: res.status,
+            canonical,
+            duplicate: samePage && named !== target,
+        });
+        if (options.delayMs) {
+            await sleep(options.delayMs);
+        }
+    });
+    return probes;
+};
+//# sourceMappingURL=site-probes.js.map

package/dist/core/site-probes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"site-probes.js","sourceRoot":"","sources":["../../src/core/site-probes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAe3E,mFAAmF;AACnF,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAQvC,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,OAAqB,EACK,EAAE;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE/C,4FAA4F;IAC5F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,QAAQ,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACtE,SAAS;YACX,CAAC;YAED,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;YAC3D,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,oGAAoG;AACpG,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,OAAqB,EACrB,KAAa,EACb,OAAqB,EACM,EAAE;IAC7B,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEjD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAErD,yFAAyF;IACzF,MAAM,OAAO,GAAG,CAAC,GAAG,YAAY,CAAC;SAC9B,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CACjB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QACnB,CAAC,OAAO,CAAC,aAAa;YACpB,UAAU,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC,GAAG,CAAC;QACP,CAAC,CAAC,EAAE,CACP;SACA,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEnB,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9D,mFAAmF;QACnF,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAE3C,MAAM,YAAY,GAAiB;YACjC,GAAG,IAAI;YACP,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;SAC/D,CAAC;QAEF,IAAI,OAAO,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACnD,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC;QACjC,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAE9C,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE;YACnB,GAAG;YACH,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AACvC,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAE7B,oGAAoG;AACpG,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,6FAA6F;AAC7F,MAAM,UAAU,GAAG,CACjB,GAAW,EAC2C,EAAE;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC5B,MAAM,GAAG,GAAyD,EAAE,CAAC;IAErE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QACrC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,GAAG,QAAQ,GAAG,CAAC;IACnB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAE1D,oEAAoE;IACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,uBAAuB,EAAE,CAAC,IAAI,EAAE,EAAE,CAClE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CACjD,CAAC;IAEF,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC;QACvB,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAkB,EAAW,EAAE,CACtD,MAAM,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG;IAC/B,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;AAE1D;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,OAAqB,EACrB,QAAgB,EAChB,OAAqB,EACO,EAAE;IAC9B,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO;SACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACZ,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAEpC,OAAO,CACL,QAAQ,KAAK,GAAG;YAChB,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,oBAAoB;YACxC,eAAe,CAAC,CAAC,CAAC;YAClB,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC;YACpB,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CACpC,CAAC;IACJ,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;IAE5B,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CACzC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC;SACnB,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAClC,CAAC;IAEF,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,MAAM,aAAa,CACjB,QAAQ,EACR,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAEpD,MAAM,QAAQ,GACZ,GAAG,CAAC,MAAM,KAAK,GAAG;YAClB,GAAG,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;YAC9B,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,cAAc,CAAC,GAAG,CAAC,CAAC;QAEtB,MAAM,WAAW,GAAG,QAAQ;YAC1B,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;YAC/C,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,SAAS,GAAG,WAAW,EAAE,SAAS,IAAI,IAAI,CAAC;QAEjD,uFAAuF;QACvF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,YAAY,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC;QAEjD,4FAA4F;QAC5F,MAAM,QAAQ,GACZ,WAAW,KAAK,IAAI;YACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC7D,SAAS,CAAC;QAEd,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO;YACP,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS;YACT,SAAS,EAAE,QAAQ,IAAI,KAAK,KAAK,MAAM;SACxC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}

package/dist/core/site-rules.d.ts

@@ -20,6 +20,18 @@ export interface LinkCheck {
     }[];
     error?: string;
 }
+/** A crawled page requested again with its trailing slash toggled, or with its path in uppercase. */
+export interface UrlVariantProbe {
+    kind: "trailing-slash" | "case";
+    /** The crawled page. */
+    url: string;
+    variant: string;
+    status: number | null;
+    /** The canonical the variant names, when it answered with a page. */
+    canonical: string | null;
+    /** The variant answered 200 itself and names no canonical that points back to the page. */
+    duplicate: boolean;
+}
 export interface ProbeResults {
     soft404?: {
         url: string;
@@ -37,6 +49,7 @@ export interface ProbeResults {
         finalUrl: string | null;
         redirectsToMain: boolean;
     } | null;
+    urlVariants?: UrlVariantProbe[];
 }
 export interface SiteContext {
     startUrl: string;
@@ -47,6 +60,8 @@ export interface SiteContext {
     /** For every internal target URL, the pages linking to it. */
     linkSources: Map<string, Set<string>>;
     robots: RobotsTxtInfo | null;
+    /** robots.txt up to 500 KiB, for checks per user agent (`robots.text` stops at 20,000 characters). */
+    robotsText?: string | null;
     robotsAllows: ((url: string) => boolean) | null;
     blockedByRobots: string[];
     sitemaps: SitemapInfo;
@@ -60,5 +75,8 @@ export interface SiteContext {
     externalSources?: Map<string, Set<string>>;
     /** Directory audits: HTML files nobody links to. */
     unlinkedPages?: string[];
+    /** Image URLs the pages show that were requested (or looked up as files), keyed by URL. */
+    imageChecks?: Map<string, LinkCheck>;
+    imageSources?: Map<string, Set<string>>;
 }
 export declare const runSiteRules: (ctx: SiteContext) => Finding[];