@forumulate/cli 0.8.0 → 0.10.0

package/dist/doctor/crawl.js

@@ -0,0 +1,210 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sitemapsFromRobots = sitemapsFromRobots;
+exports.readSitemaps = readSitemaps;
+exports.sample = sample;
+exports.sitemapCrawl = sitemapCrawl;
+exports.groupOf = groupOf;
+exports.cacheClass = cacheClass;
+exports.cacheConsistency = cacheConsistency;
+const probe_1 = require("./probe");
+/**
+ * `sitemap-crawl`: every address the site hands a crawler, fetched.
+ *
+ * The sitemap is the list a search engine trusts most, so an entry that
+ * redirects, 404s, names a different canonical or sits on another host is a
+ * page the site is asking to have indexed and then refusing to be. A space root
+ * that 307s to its first page was listed as itself (FMU-B06); a platform copy
+ * that 308s elsewhere named itself canonical (FMU-B01). Each is invisible from
+ * the page a person happens to open, and obvious once every listed URL is asked.
+ *
+ * The pages fetched here are also the SAMPLE the other site-wide checks judge
+ * (cache headers, descriptions), so the crawl is done once.
+ */
+/** How many sitemap documents (index included) one run will read. */
+const SITEMAP_DOC_LIMIT = 25;
+/** `Sitemap:` lines of robots.txt, else the conventional address. */
+function sitemapsFromRobots(robots, origin) {
+    const named = [...robots.matchAll(/^\s*sitemap:\s*(\S+)/gim)].map((m) => m[1]);
+    return named.length ? named : [`${origin}/sitemap.xml`];
+}
+function locs(xml) {
+    return [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => m[1].replace(/&amp;/g, '&'));
+}
+/**
+ * Read sitemaps (following a sitemap index), returning the URL lists per
+ * document, so the sample can draw from every space rather than exhaust the
+ * first one: a site with a 900-page reference and a 4-page legal space would
+ * otherwise never have its legal pages looked at.
+ */
+async function readSitemaps(get, start) {
+    const lists = [];
+    const problems = [];
+    const queue = [...new Set(start)];
+    const seen = new Set();
+    while (queue.length && seen.size < SITEMAP_DOC_LIMIT) {
+        const url = queue.shift();
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        const p = await get(url, { accept: 'application/xml,text/xml' });
+        if (!p.ok || p.status !== 200) {
+            problems.push(`${url} → ${(0, probe_1.describeProbe)(p)}`);
+            continue;
+        }
+        if (/<sitemapindex\b/i.test(p.text))
+            queue.push(...locs(p.text));
+        else
+            lists.push({ sitemap: url, urls: locs(p.text) });
+    }
+    return { lists, problems };
+}
+/** Up to `limit` URLs, round-robin across the sitemaps so every space is represented. */
+function sample(lists, limit) {
+    const out = [];
+    const seen = new Set();
+    for (let i = 0; out.length < limit; i++) {
+        let any = false;
+        for (const l of lists) {
+            const u = l.urls[i];
+            if (u === undefined)
+                continue;
+            any = true;
+            if (!seen.has(u)) {
+                seen.add(u);
+                out.push(u);
+                if (out.length >= limit)
+                    break;
+            }
+        }
+        if (!any)
+            break;
+    }
+    return out;
+}
+async function sitemapCrawl(get, origin, robots, limit) {
+    const title = 'Every URL in the sitemap answers 200, as itself';
+    const matters = 'A sitemap entry that redirects, 404s or names another canonical is a page the site asks to have indexed and then disowns; search engines lower their trust in the whole list.';
+    const { lists, problems } = await readSitemaps(get, sitemapsFromRobots(robots ?? '', origin));
+    if (lists.length === 0) {
+        return {
+            check: (0, probe_1.check)('sitemap-crawl', title, 'fail', problems.length ? `No sitemap could be read: ${(0, probe_1.some)(problems)}` : 'No sitemap found.', matters),
+            pages: [],
+        };
+    }
+    const total = lists.reduce((n, l) => n + l.urls.length, 0);
+    const urls = sample(lists, limit);
+    const siteHost = (0, probe_1.hostOf)(origin);
+    const pages = await (0, probe_1.pool)(urls, 6, (u) => get(u, { accept: 'text/html' }));
+    const bad = [...problems];
+    for (const p of pages) {
+        if ((0, probe_1.hostOf)(p.url) !== siteHost) {
+            bad.push(`${p.url} is on another host than the sitemap (${siteHost})`);
+            continue;
+        }
+        if (!p.ok) {
+            bad.push(`${p.url} → ${p.error}`);
+            continue;
+        }
+        if (p.status >= 300 && p.status < 400) {
+            bad.push(`${p.url} → ${p.status} ${p.headers.get('location') ?? ''} (list the final URL)`);
+            continue;
+        }
+        if (p.status !== 200) {
+            bad.push(`${p.url} → ${p.status}`);
+            continue;
+        }
+        const canonical = (0, probe_1.canonicalOf)(p.text, p.url);
+        if (canonical && !(0, probe_1.sameUrl)(canonical, p.url))
+            bad.push(`${p.url} names canonical ${canonical}`);
+    }
+    const scope = `${pages.length} of ${total} listed URL${total === 1 ? '' : 's'} across ${lists.length} sitemap${lists.length === 1 ? '' : 's'}`;
+    return {
+        check: (0, probe_1.check)('sitemap-crawl', title, bad.length ? 'fail' : 'pass', bad.length
+            ? `${bad.length} problem(s) in ${scope}: ${(0, probe_1.some)(bad)}`
+            : `${scope}: all 200, all self-canonical.`, bad.length ? matters : ''),
+        pages,
+    };
+}
+/**
+ * The space a URL belongs to, for grouping: its first path segment, which is
+ * where a path-mounted space lives (`/docs`, `/legal`). Pages at the root of a
+ * site domain group together under `/`. Coarse, and that is fine — a
+ * difference between two spaces is a choice; a difference inside one is what
+ * this is looking for.
+ */
+function groupOf(url) {
+    try {
+        const u = new URL(url);
+        const segs = u.pathname.split('/').filter(Boolean);
+        // A short reader prefix followed by owner and space (`/p/<org>/<space>/…`)
+        // is how a multi-tenant host addresses a space; grouping by `/p` alone
+        // would compare every tenant's pages with every other's.
+        if (segs.length >= 4 && segs[0].length <= 3)
+            return `${u.host}/${segs.slice(0, 3).join('/')}`;
+        return `${u.host}/${segs[0] ?? ''}`;
+    }
+    catch {
+        return url;
+    }
+}
+/** `public` (a shared cache may keep it) or `private` (it may not), by the directives that decide it. */
+function cacheClass(cc) {
+    if (!cc)
+        return 'none';
+    if (/\b(?:private|no-store)\b/i.test(cc))
+        return 'private';
+    if (/\b(?:public|s-maxage)\b/i.test(cc))
+        return 'public';
+    return 'private';
+}
+/** Headers a site may send to say a page is uncached on purpose (and why). */
+const REASON_HEADERS = ['x-forumulate-cache-reason', 'x-cache-reason', 'x-cache-control-reason'];
+/**
+ * `cache-consistency`: the pages of one space are cached the same way.
+ *
+ * Every legal document answered `private, no-cache, no-store` to a reader with
+ * no cookie at all, while the legal index beside it was `public, s-maxage=10`
+ * (FMU-B05). Neither header is wrong on its own; the difference is the defect —
+ * either something is gated and should say so, or a code path forgot the
+ * public header and every reader pays a full origin render. A page that sends
+ * a reason header is taken at its word.
+ */
+function cacheConsistency(pages) {
+    const title = 'Pages within a space are cached alike';
+    const groups = new Map();
+    for (const p of pages) {
+        if (!p.ok || p.status !== 200)
+            continue;
+        if (REASON_HEADERS.some((h) => p.headers.get(h)))
+            continue;
+        const cc = p.headers.get('cache-control');
+        const g = groupOf(p.url);
+        groups.set(g, [
+            ...(groups.get(g) ?? []),
+            { url: p.url, cls: cacheClass(cc), cc: cc ?? '(none)' },
+        ]);
+    }
+    if (groups.size === 0)
+        return (0, probe_1.check)('cache-consistency', title, 'skip', 'No sampled pages answered 200.', '');
+    const odd = [];
+    for (const [g, rows] of groups) {
+        const classes = [...new Set(rows.map((r) => r.cls))];
+        if (classes.length < 2)
+            continue;
+        // Every side of the split is named. Which side is the bug is the site's
+        // call (usually the private one — a published page has no reason to be),
+        // and a "majority" of three legal documents is evidence of nothing.
+        const sides = classes.map((cls) => {
+            const of = rows.filter((r) => r.cls === cls);
+            return `${of.length} ${cls} ("${of[0].cc}": ${(0, probe_1.some)(of.map((r) => r.url), 3)})`;
+        });
+        odd.push(`${g}: ${sides.join(' vs ')}`);
+    }
+    return (0, probe_1.check)('cache-consistency', title, odd.length ? 'fail' : 'pass', odd.length
+        ? (0, probe_1.some)(odd)
+        : `${[...groups.values()].reduce((n, r) => n + r.length, 0)} pages in ${groups.size} space(s), one rule each.`, odd.length
+        ? 'A published page answering no-store to an anonymous reader is rendered at the origin for every visit. If it is gated on purpose, say so with an x-forumulate-cache-reason header.'
+        : '');
+}
+//# sourceMappingURL=crawl.js.map

package/dist/doctor/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/doctor/crawl.ts"],"names":[],"mappings":";;AAqCA,gDAGC;AAcD,oCAqBC;AAGD,wBAkBC;AAED,oCA4DC;AASD,0BAYC;AAGD,gCAKC;AAeD,4CA0CC;AAnPD,mCAUiB;AAEjB;;;;;;;;;;;;GAYG;AAEH,qEAAqE;AACrE,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAQ7B,qEAAqE;AACrE,SAAgB,kBAAkB,CAAC,MAAc,EAAE,MAAc;IAC/D,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IAChF,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClE,CAAC,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,YAAY,CAChC,GAAQ,EACR,KAAe;IAEf,MAAM,KAAK,GAA0C,EAAE,CAAC;IACxD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,GAAG,iBAAiB,EAAE,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC9B,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9C,SAAS;QACX,CAAC;QACD,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;;YAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,yFAAyF;AACzF,SAAgB,MAAM,CAAC,KAA2B,EAAE,KAAa;IAC/D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,KAAK,SAAS;gBAAE,SAAS;YAC9B,GAAG,GAAG,IAAI,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACZ,IAAI,GAAG,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAM;YACjC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG;YAAE,MAAM;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAEM,KAAK,UAAU,YAAY,CAChC,GAAQ,EACR,MAAc,EACd,MAA0B,EAC1B,KAAa;IAEb,MAAM,KAAK,GAAG,iDAAiD,CAAC;IAChE,MAAM,OAAO,GACX,+KAA+K,CAAC;IAClL,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,kBAAkB,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,KAAK,EAAE,IAAA,aAAK,EACV,eAAe,EACf,KAAK,EACL,MAAM,EACN,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B,IAAA,YAAI,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EACrF,OAAO,CACR;YACD,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAA,cAAM,EAAC,MAAM,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,IAAA,YAAI,EAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAa,CAAC,GAAG,QAAQ,CAAC,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAA,cAAM,EAAC,CAAC,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,yCAAyC,QAAQ,GAAG,CAAC,CAAC;YACvE,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAClC,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;YAC3F,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,IAAA,mBAAW,EAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,SAAS,IAAI,CAAC,IAAA,eAAO,EAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,oBAAoB,SAAS,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,MAAM,OAAO,KAAK,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,MAAM,WAAW,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC/I,OAAO;QACL,KAAK,EAAE,IAAA,aAAK,EACV,eAAe,EACf,KAAK,EACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAC5B,GAAG,CAAC,MAAM;YACR,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,kBAAkB,KAAK,KAAK,IAAA,YAAI,EAAC,GAAG,CAAC,EAAE;YACtD,CAAC,CAAC,GAAG,KAAK,gCAAgC,EAC5C,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC1B;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,OAAO,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnD,2EAA2E;QAC3E,uEAAuE;QACvE,yDAAyD;QACzD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/F,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,yGAAyG;AACzG,SAAgB,UAAU,CAAC,EAAiB;IAC1C,IAAI,CAAC,EAAE;QAAE,OAAO,MAAM,CAAC;IACvB,IAAI,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3D,IAAI,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,QAAQ,CAAC;IACzD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,MAAM,cAAc,GAAG,CAAC,2BAA2B,EAAE,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;AAEjG;;;;;;;;;GASG;AACH,SAAgB,gBAAgB,CAAC,KAAc;IAC7C,MAAM,KAAK,GAAG,uCAAuC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsD,CAAC;IAC7E,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG;YAAE,SAAS;QACxC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QAC3D,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;YACZ,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACxB,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,QAAQ,EAAE;SACxD,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QACnB,OAAO,IAAA,aAAK,EAAC,mBAAmB,EAAE,KAAK,EAAE,MAAM,EAAE,gCAAgC,EAAE,EAAE,CAAC,CAAC;IACzF,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACjC,wEAAwE;QACxE,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YAC7C,OAAO,GAAG,EAAE,CAAC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,CAAC,CAAE,CAAC,EAAE,MAAM,IAAA,YAAI,EACjD,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EACpB,CAAC,CACF,GAAG,CAAC;QACP,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAA,aAAK,EACV,mBAAmB,EACnB,KAAK,EACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAC5B,GAAG,CAAC,MAAM;QACR,CAAC,CAAC,IAAA,YAAI,EAAC,GAAG,CAAC;QACX,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,aAAa,MAAM,CAAC,IAAI,2BAA2B,EAChH,GAAG,CAAC,MAAM;QACR,CAAC,CAAC,mLAAmL;QACrL,CAAC,CAAC,EAAE,CACP,CAAC;AACJ,CAAC"}

package/dist/doctor/extended.d.ts

@@ -0,0 +1,52 @@
+import type { Check } from '../conformance';
+import { type AddressSet } from './address-checks';
+import { type Get } from './probe';
+/**
+ * The site-wide checks added for FMU-D14, run after the single-page ones.
+ *
+ * Every one of them is a defect an integrator found on a live site after
+ * readers did, and every one is judged from the outside, by HTTP alone, so the
+ * report is the same whether it is pointed at our platform, a staging mount or
+ * any other docs host. The only thing a check knows about "the platform" is
+ * its hostname, which is an input (`--platform-host`, else the host of
+ * FORUMULATE_URL, else forumulate.com) — never a constant in the logic.
+ */
+/** Every check id this file can produce, in report order. `--only` validates against it. */
+export declare const EXTENDED_CHECK_IDS: readonly ["canonical-both-addresses", "sitemap-crawl", "cache-consistency", "root-redirect", "meta-description", "llms-txt", "favicon", "consent-tag", "white-label-leaks", "robots-params"];
+export declare const DEFAULT_PLATFORM_HOST = "forumulate.com";
+/** URLs one run fetches from the sitemaps; enough to see a pattern, few enough for CI. */
+export declare const DEFAULT_SAMPLE_LIMIT = 60;
+export interface ExtendedInput {
+    /** The page judged: where the asked-for address landed after redirects. */
+    url: string;
+    /** The address as given, before any redirect. */
+    requestedUrl?: string;
+    /** The page's HTML as an anonymous reader got it ('' when it did not answer). */
+    html: string;
+    /** robots.txt, when it answered 200. */
+    robots?: string;
+    get: Get;
+    platformHost: string;
+    /** The platform's API base (`https://forumulate.com/api`), for the primary-address lookup. */
+    platformApi?: string;
+    /** Other addresses of this page/space, from `--also` or `--site`. */
+    addressSets?: AddressSet[];
+    /** Space roots to check, from `--site`; derived from the URL otherwise. */
+    roots?: string[];
+    limit?: number;
+    /** Run only these ids (undefined: all). */
+    only?: ReadonlySet<string>;
+}
+/** The roots a URL's space is served at: the mount (first segment), or the platform copy's space root. */
+export declare function deriveRoots(url: string, platformHost: string): string[];
+/**
+ * The customer's address of a platform-hosted page, asked of the platform.
+ *
+ * Only for a URL on the platform host in the platform-copy shape: the public
+ * primary-domain lookup answers where the space's readers are sent, which is
+ * what both copies must name as canonical. Any failure is "unknown", and the
+ * check then compares the copies it has with each other.
+ */
+export declare function discoverPrimary(get: Get, url: string, platformHost: string, platformApi: string | undefined): Promise<AddressSet | undefined>;
+export declare function extendedChecks(input: ExtendedInput): Promise<Check[]>;
+//# sourceMappingURL=extended.d.ts.map

package/dist/doctor/extended.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extended.d.ts","sourceRoot":"","sources":["../../src/doctor/extended.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAiB,KAAK,GAAG,EAAc,MAAM,SAAS,CAAC;AAG9D;;;;;;;;;GASG;AAEH,4FAA4F;AAC5F,eAAO,MAAM,kBAAkB,8LAWrB,CAAC;AAEX,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AACtD,0FAA0F;AAC1F,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,MAAM,WAAW,aAAa;IAC5B,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,iDAAiD;IACjD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,GAAG,CAAC;IACT,YAAY,EAAE,MAAM,CAAC;IACrB,8FAA8F;IAC9F,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,IAAI,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC5B;AASD,0GAA0G;AAC1G,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,EAAE,CASvE;AAED;;;;;;;GAOG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,MAAM,EACX,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,GAAG,SAAS,GAC9B,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAuBjC;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAiF3E"}

package/dist/doctor/extended.js

@@ -0,0 +1,150 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_SAMPLE_LIMIT = exports.DEFAULT_PLATFORM_HOST = exports.EXTENDED_CHECK_IDS = void 0;
+exports.deriveRoots = deriveRoots;
+exports.discoverPrimary = discoverPrimary;
+exports.extendedChecks = extendedChecks;
+const address_checks_1 = require("./address-checks");
+const crawl_1 = require("./crawl");
+const page_checks_1 = require("./page-checks");
+const probe_1 = require("./probe");
+const robots_params_1 = require("./robots-params");
+/**
+ * The site-wide checks added for FMU-D14, run after the single-page ones.
+ *
+ * Every one of them is a defect an integrator found on a live site after
+ * readers did, and every one is judged from the outside, by HTTP alone, so the
+ * report is the same whether it is pointed at our platform, a staging mount or
+ * any other docs host. The only thing a check knows about "the platform" is
+ * its hostname, which is an input (`--platform-host`, else the host of
+ * FORUMULATE_URL, else forumulate.com) — never a constant in the logic.
+ */
+/** Every check id this file can produce, in report order. `--only` validates against it. */
+exports.EXTENDED_CHECK_IDS = [
+    'canonical-both-addresses',
+    'sitemap-crawl',
+    'cache-consistency',
+    'root-redirect',
+    'meta-description',
+    'llms-txt',
+    'favicon',
+    'consent-tag',
+    'white-label-leaks',
+    'robots-params',
+];
+exports.DEFAULT_PLATFORM_HOST = 'forumulate.com';
+/** URLs one run fetches from the sitemaps; enough to see a pattern, few enough for CI. */
+exports.DEFAULT_SAMPLE_LIMIT = 60;
+/**
+ * A platform copy's address shape: `/<prefix>/<org>/<space>/<rest>`, where the
+ * prefix is a short reader route (`/p/`, `/b/`, `/f/` …). Read only on the
+ * platform host, where that shape is the platform's own convention.
+ */
+const PLATFORM_COPY = /^\/([a-z]{1,3})\/([^/]+)\/([^/]+)(\/.*)?$/;
+/** The roots a URL's space is served at: the mount (first segment), or the platform copy's space root. */
+function deriveRoots(url, platformHost) {
+    const u = new URL(url);
+    const path = u.pathname.replace(/\/+$/, '');
+    if ((0, probe_1.hostOf)(url) === platformHost.toLowerCase()) {
+        const m = PLATFORM_COPY.exec(path);
+        if (m)
+            return [`${u.origin}/${m[1]}/${m[2]}/${m[3]}`];
+    }
+    const first = path.split('/').filter(Boolean)[0];
+    return [first ? `${u.origin}/${first}` : u.origin];
+}
+/**
+ * The customer's address of a platform-hosted page, asked of the platform.
+ *
+ * Only for a URL on the platform host in the platform-copy shape: the public
+ * primary-domain lookup answers where the space's readers are sent, which is
+ * what both copies must name as canonical. Any failure is "unknown", and the
+ * check then compares the copies it has with each other.
+ */
+async function discoverPrimary(get, url, platformHost, platformApi) {
+    if (!platformApi || (0, probe_1.hostOf)(url) !== platformHost.toLowerCase())
+        return undefined;
+    const u = new URL(url);
+    const m = PLATFORM_COPY.exec(u.pathname.replace(/\/+$/, ''));
+    if (!m)
+        return undefined;
+    const q = new URLSearchParams({
+        org: decodeURIComponent(m[2]),
+        space: decodeURIComponent(m[3]),
+    });
+    const p = await get(`${platformApi.replace(/\/+$/, '')}/public/domains/primary?${q}`, {
+        accept: 'application/json',
+    });
+    if (!p.ok || p.status !== 200)
+        return undefined;
+    try {
+        const primary = JSON.parse(p.text).primary;
+        if (!primary?.origin)
+            return undefined;
+        const expected = `${primary.origin.replace(/\/+$/, '')}${primary.basePath ?? ''}${m[4] ?? ''}`;
+        return { expected, urls: [url, expected] };
+    }
+    catch {
+        return undefined;
+    }
+}
+async function extendedChecks(input) {
+    const want = (id) => !input.only || input.only.has(id);
+    const { get, url, html, platformHost } = input;
+    const origin = new URL(url).origin;
+    const roots = input.roots?.length
+        ? input.roots
+        : deriveRoots(input.requestedUrl ?? url, platformHost);
+    const out = [];
+    if (want('canonical-both-addresses')) {
+        const sets = [...(input.addressSets ?? [])];
+        if (sets.length === 0) {
+            // The address the user GAVE: a platform copy that redirects (the legal
+            // page went `/p/<org>/legal/<doc>` → `/legal/<org>/<doc>`) has lost the
+            // org/space shape by the time it lands.
+            const asked = input.requestedUrl ?? url;
+            const found = await discoverPrimary(get, asked, platformHost, input.platformApi);
+            if (found)
+                sets.push(found);
+        }
+        out.push(await (0, address_checks_1.canonicalBothAddresses)(get, sets));
+    }
+    // One crawl feeds three checks; it is skipped only when none of them is asked for.
+    let pages = [];
+    if (want('sitemap-crawl') || want('cache-consistency') || want('meta-description')) {
+        const crawl = await (0, crawl_1.sitemapCrawl)(get, origin, input.robots, input.limit ?? exports.DEFAULT_SAMPLE_LIMIT);
+        pages = crawl.pages;
+        if (want('sitemap-crawl'))
+            out.push(crawl.check);
+    }
+    if (want('cache-consistency'))
+        out.push((0, crawl_1.cacheConsistency)(pages));
+    if (want('root-redirect'))
+        out.push(await (0, page_checks_1.rootRedirect)(get, roots));
+    if (want('meta-description')) {
+        // The page that was asked about is judged too, even when no sitemap lists it.
+        const self = html
+            ? [{ ok: true, url, status: 200, headers: new Headers(), text: html }]
+            : [];
+        const listed = pages.some((p) => p.url === url);
+        out.push((0, page_checks_1.metaDescription)(listed ? pages : [...self, ...pages]));
+    }
+    if (want('llms-txt'))
+        out.push(await (0, page_checks_1.llmsTxt)(get, origin, roots.filter((r) => r !== origin), platformHost));
+    if (want('favicon')) {
+        out.push(html
+            ? await (0, page_checks_1.favicon)(get, url, html, roots.filter((r) => r !== origin))
+            : (0, probe_1.check)('favicon', 'The favicon resolves to an image', 'skip', 'No HTML to read.', ''));
+    }
+    if (want('consent-tag'))
+        out.push(html
+            ? (0, address_checks_1.consentTag)(html)
+            : (0, probe_1.check)('consent-tag', 'The consent banner is told the page language', 'skip', 'No HTML to read.', ''));
+    if (want('white-label-leaks'))
+        out.push(await (0, address_checks_1.whiteLabelLeaks)(get, url, html, platformHost));
+    // SEO §1: the Disallow lines for each mount's sorted / filtered copies.
+    if (want('robots-params'))
+        out.push((0, robots_params_1.robotsParams)(roots, origin, input.robots));
+    return out;
+}
+//# sourceMappingURL=extended.js.map

package/dist/doctor/extended.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extended.js","sourceRoot":"","sources":["../../src/doctor/extended.ts"],"names":[],"mappings":";;;AAuEA,kCASC;AAUD,0CA4BC;AAED,wCAiFC;AAxMD,qDAK0B;AAC1B,mCAAyD;AACzD,+CAAgF;AAChF,mCAA8D;AAC9D,mDAA+C;AAE/C;;;;;;;;;GASG;AAEH,4FAA4F;AAC/E,QAAA,kBAAkB,GAAG;IAChC,0BAA0B;IAC1B,eAAe;IACf,mBAAmB;IACnB,eAAe;IACf,kBAAkB;IAClB,UAAU;IACV,SAAS;IACT,aAAa;IACb,mBAAmB;IACnB,eAAe;CACP,CAAC;AAEE,QAAA,qBAAqB,GAAG,gBAAgB,CAAC;AACtD,0FAA0F;AAC7E,QAAA,oBAAoB,GAAG,EAAE,CAAC;AAwBvC;;;;GAIG;AACH,MAAM,aAAa,GAAG,2CAA2C,CAAC;AAElE,0GAA0G;AAC1G,SAAgB,WAAW,CAAC,GAAW,EAAE,YAAoB;IAC3D,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,IAAA,cAAM,EAAC,GAAG,CAAC,KAAK,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,eAAe,CACnC,GAAQ,EACR,GAAW,EACX,YAAoB,EACpB,WAA+B;IAE/B,IAAI,CAAC,WAAW,IAAI,IAAA,cAAM,EAAC,GAAG,CAAC,KAAK,YAAY,CAAC,WAAW,EAAE;QAAE,OAAO,SAAS,CAAC;IACjF,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,MAAM,CAAC,GAAG,IAAI,eAAe,CAAC;QAC5B,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC;QAC9B,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC;KACjC,CAAC,CAAC;IACH,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,2BAA2B,CAAC,EAAE,EAAE;QACpF,MAAM,EAAE,kBAAkB;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,CAAC;QACH,MAAM,OAAO,GACX,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAClB,CAAC,OAAO,CAAC;QACV,IAAI,CAAC,OAAO,EAAE,MAAM;YAAE,OAAO,SAAS,CAAC;QACvC,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/F,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,KAAoB;IACvD,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,MAAM;QAC/B,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,IAAI,GAAG,EAAE,YAAY,CAAC,CAAC;IACzD,MAAM,GAAG,GAAY,EAAE,CAAC;IAExB,IAAI,IAAI,CAAC,0BAA0B,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,uEAAuE;YACvE,wEAAwE;YACxE,wCAAwC;YACxC,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,IAAI,GAAG,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;YACjF,IAAI,KAAK;gBAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,MAAM,IAAA,uCAAsB,EAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,mFAAmF;IACnF,IAAI,KAAK,GAAY,EAAE,CAAC;IACxB,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAY,EAC9B,GAAG,EACH,MAAM,EACN,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,KAAK,IAAI,4BAAoB,CACpC,CAAC;QACF,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACpB,IAAI,IAAI,CAAC,eAAe,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,IAAI,CAAC,mBAAmB,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,IAAA,wBAAgB,EAAC,KAAK,CAAC,CAAC,CAAC;IACjE,IAAI,IAAI,CAAC,eAAe,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAA,0BAAY,EAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;IACpE,IAAI,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC7B,8EAA8E;QAC9E,MAAM,IAAI,GAAY,IAAI;YACxB,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACtE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;QAChD,GAAG,CAAC,IAAI,CAAC,IAAA,6BAAe,EAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,CAAC;QAClB,GAAG,CAAC,IAAI,CACN,MAAM,IAAA,qBAAO,EACX,GAAG,EACH,MAAM,EACN,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,EACjC,YAAY,CACb,CACF,CAAC;IACJ,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpB,GAAG,CAAC,IAAI,CACN,IAAI;YACF,CAAC,CAAC,MAAM,IAAA,qBAAO,EACX,GAAG,EACH,GAAG,EACH,IAAI,EACJ,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAClC;YACH,CAAC,CAAC,IAAA,aAAK,EAAC,SAAS,EAAE,kCAAkC,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CACzF,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,CAAC;QACrB,GAAG,CAAC,IAAI,CACN,IAAI;YACF,CAAC,CAAC,IAAA,2BAAU,EAAC,IAAI,CAAC;YAClB,CAAC,CAAC,IAAA,aAAK,EACH,aAAa,EACb,8CAA8C,EAC9C,MAAM,EACN,kBAAkB,EAClB,EAAE,CACH,CACN,CAAC;IACJ,IAAI,IAAI,CAAC,mBAAmB,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAA,gCAAe,EAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;IAC7F,wEAAwE;IACxE,IAAI,IAAI,CAAC,eAAe,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,IAAA,4BAAY,EAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/E,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/doctor/page-checks.d.ts

@@ -0,0 +1,42 @@
+import type { Check } from '../conformance';
+import { type Get, type Probe } from './probe';
+/**
+ * `root-redirect`: a space's root either renders or moves permanently, cheaply.
+ *
+ * `GET /docs` answered `307` with a 66 KB HTML body and a canonical naming
+ * `/docs` (FMU-B06). A 307 says "this is temporary, keep asking me", so a
+ * crawler keeps the root as the indexed URL and never settles on the page it
+ * lands on; the body is bytes every visitor downloads and throws away. A 301
+ * or 308 with an empty body, or the front page rendered in place, are both fine.
+ */
+export declare function rootRedirect(get: Get, roots: string[]): Promise<Check>;
+/**
+ * `meta-description`: every page says what it is, in words somebody wrote.
+ *
+ * Checked across the crawl sample, not just the page given, because the
+ * preset fallbacks live on the pages nobody opens while testing — a space
+ * index, a blog home, a generated reference landing (FMU-B08, FMU-B09).
+ */
+export declare function metaDescription(pages: Probe[]): Check;
+/**
+ * `favicon`: the icons the page and the host advertise are images.
+ *
+ * `GET /docs/favicon.ico` on a customer's domain answered a 404 with a 104 KB
+ * HTML page (FMU-B17). Browsers, feed readers and link unfurlers ask for these
+ * addresses on their own; a 100 KB error page per ask is load, and a missing
+ * icon is a blank tab. An asset-shaped path that is not there should be a tiny
+ * 404, not the site's whole not-found page.
+ */
+export declare function favicon(get: Get, pageUrl: string, html: string, roots: string[]): Promise<Check>;
+/** One llms.txt judged against the llmstxt.org shape. */
+export declare function llmsProblems(text: string, url: string, platformHost?: string): string[];
+/**
+ * `llms-txt`: the file an AI agent reads first exists and is shaped for one.
+ *
+ * The space's llms.txt opened with "Published documentation for API Docs.",
+ * listed pages in recent-edit order with no descriptions, and titled a clash
+ * "Authentication (ref-authentication)" (FMU-B16). An agent picks pages from
+ * this list by their descriptions; without them it fetches at random.
+ */
+export declare function llmsTxt(get: Get, origin: string, roots: string[], platformHost?: string): Promise<Check>;
+//# sourceMappingURL=page-checks.d.ts.map

package/dist/doctor/page-checks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-checks.d.ts","sourceRoot":"","sources":["../../src/doctor/page-checks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAE5C,OAAO,EAUL,KAAK,GAAG,EACR,KAAK,KAAK,EACX,MAAM,SAAS,CAAC;AAKjB;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CA4C5E;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,KAAK,CAuBrD;AAID;;;;;;;;GAQG;AACH,wBAAsB,OAAO,CAC3B,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EAAE,GACd,OAAO,CAAC,KAAK,CAAC,CAyChB;AAKD,yDAAyD;AACzD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CA+BvF;AAED;;;;;;;GAOG;AACH,wBAAsB,OAAO,CAC3B,GAAG,EAAE,GAAG,EACR,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EAAE,EACf,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,KAAK,CAAC,CA8BhB"}