@forumulate/cli 0.9.0 → 0.10.0

package/dist/doctor/command.js

@@ -0,0 +1,234 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DOCTOR_HELP = exports.DOCTOR_CHECK_IDS = void 0;
+exports.runDoctorCommand = runDoctorCommand;
+exports.printReport = printReport;
+const args_1 = require("../api/args");
+const http_1 = require("../api/http");
+const api_base_1 = require("../api-base");
+const config_1 = require("../commands/config");
+const credentials_1 = require("../credentials");
+const exit_codes_1 = require("../exit-codes");
+const doctor_1 = require("../doctor");
+const extended_1 = require("./extended");
+const site_1 = require("./site");
+const probe_1 = require("./probe");
+const reader_auth_atlas_1 = require("./reader-auth-atlas");
+exports.DOCTOR_CHECK_IDS = [
+    ...doctor_1.BASE_CHECK_IDS,
+    ...extended_1.EXTENDED_CHECK_IDS,
+    reader_auth_atlas_1.READER_AUTH_ATLAS_CHECK,
+];
+exports.DOCTOR_HELP = `forumulate doctor — check a live site from the outside
+
+Usage:
+  forumulate doctor <readerUrl> [flags]
+  forumulate doctor --site <slug|id> [--org <slug|id>] [flags]
+
+Plain HTTP, no token needed for a URL: the same report against our platform, a
+staging mount or any other docs host. --site asks the API for every address of
+every space on a site (needs a token), and checks each copy.
+
+Flags:
+  --site <slug|id>     every member space, at its custom domain AND its
+                       platform copy (token: FORUMULATE_TOKEN or \`login\`)
+  --org <slug|id>      the org --site is in (default: FORUMULATE_ORG / config)
+  --also <url>         another address of the same page, e.g. the platform copy
+                       of a custom-domain page (repeatable)
+  --only a,b           run just these checks (ids below)
+  --json               the full report as JSON on stdout (nothing else there)
+  --api-base <origin>  where /api/health lives, when not on the page's host
+  --platform-host <h>  the hosting platform's hostname, for white-label and
+                       canonical checks (default: FORUMULATE_URL's host, else
+                       ${extended_1.DEFAULT_PLATFORM_HOST})
+  --limit <n>          sitemap URLs to sample (default ${extended_1.DEFAULT_SAMPLE_LIMIT})
+  --timeout <ms>       per-request timeout (default 15000)
+
+Checks (id — what fails it):
+  health                    /api/health not 2xx JSON (also tried under the mount)
+  public-read               a logged-out reader is sent to sign-in
+  canonical                 the page's canonical names another host
+  html-untouched            a proxy rewrites the markup (email obfuscation)
+  cache-headers             no Cache-Control, or no-store, on the page
+  robots / sitemap          robots.txt missing, disallows the path, no Sitemap:
+  crawler-access            a crawler gets a different status than a reader
+  feeds                     an advertised feed does not answer 200
+  images                    a picture on the page does not load
+  canonical-both-addresses  a page's copies (custom domain, platform) name
+                            different canonicals, or not the primary address
+  sitemap-crawl             a sitemap URL redirects, 404s, is on another host or
+                            names another canonical
+  cache-consistency         pages of one space cached differently (public vs
+                            private/no-store) with no x-forumulate-cache-reason
+  root-redirect             a space root answers 302/307, or a redirect with a
+                            body over 1 KB, or lands on a page canonical elsewhere
+  meta-description          missing description/og:description, template text
+                            ("Guides and reference, in a nested tree."), or
+                            headings run together
+  llms-txt                  /llms.txt missing, no H1, preset summary, links on
+                            another host, slugs in titles, no sections/descriptions
+  favicon                   /favicon.ico or <link rel=icon> not an image; an
+                            asset path answering a large HTML page
+  consent-tag               a consent manager (any CMP: Cookie Munch, Cookiebot,
+                            OneTrust, Osano, Termly, Usercentrics, Didomi,
+                            iubenda, Complianz, TCF) not told the page language
+  white-label-leaks         a custom-domain page or its .md twin links to the
+                            platform host
+  robots-params             (advice) prints the Disallow lines that keep crawlers
+                            off each mount's ?sort= / ?period= / ?range= copies
+  reader-auth-atlas         (advice, needs --org and a token) the organization
+                            uses Atlas but signs readers in another way (an
+                            external_jwt route): prints the relink and the switch
+                            to the Atlas reader provider (readerAuth: atlas)
+
+Examples:
+  forumulate doctor https://docs.acme.com/docs
+  forumulate doctor https://acme.com/legal/privacy --also https://forumulate.com/p/acme/legal/privacy
+  forumulate doctor --site main --org acme --json > doctor.json
+  forumulate doctor https://acme.com/docs --only sitemap-crawl,root-redirect
+
+In CI: packages/cli/examples/github-action-doctor.yml runs this on a schedule and
+on every push, uploads the JSON report and fails the job on any FAIL.
+
+${exit_codes_1.EXIT_CODES_HELP}
+  doctor: 1 when any check FAILs; a SKIP is not a failure.`;
+const SWITCHES = new Set(['json', 'help', 'h']);
+/** `forumulate doctor …` → an exit code. */
+async function runDoctorCommand(argv, deps) {
+    const p = (0, args_1.parseArgs)(argv, SWITCHES);
+    if ((0, args_1.has)(p, 'help') || (0, args_1.has)(p, 'h')) {
+        deps.out(exports.DOCTOR_HELP);
+        return exit_codes_1.EXIT.OK;
+    }
+    const env = deps.env ?? process.env;
+    const json = (0, args_1.has)(p, 'json');
+    const usage = (why) => {
+        deps.err(why);
+        deps.err('Usage: forumulate doctor <readerUrl> | --site <slug|id> [--org …]  (doctor --help)');
+        return exit_codes_1.EXIT.USAGE;
+    };
+    const only = (0, args_1.all)(p, 'only')
+        .flatMap((v) => v.split(','))
+        .map((v) => v.trim())
+        .filter(Boolean);
+    const unknown = only.filter((id) => !exports.DOCTOR_CHECK_IDS.includes(id));
+    if (unknown.length)
+        return usage(`Unknown check id(s): ${unknown.join(', ')}. Known: ${exports.DOCTOR_CHECK_IDS.join(', ')}.`);
+    const limit = (0, args_1.flag)(p, 'limit') === undefined ? undefined : Number((0, args_1.flag)(p, 'limit'));
+    if (limit !== undefined && !(Number.isInteger(limit) && limit > 0))
+        return usage('--limit takes a positive whole number.');
+    const timeout = (0, args_1.flag)(p, 'timeout') === undefined ? undefined : Number((0, args_1.flag)(p, 'timeout'));
+    if (timeout !== undefined && !(Number.isInteger(timeout) && timeout > 0))
+        return usage('--timeout takes a positive number of milliseconds.');
+    let url = p.positionals[0];
+    if (p.positionals.length > 1)
+        return usage(`One URL at a time; got ${p.positionals.join(' ')}.`);
+    if (url !== undefined) {
+        try {
+            const parsed = new URL(url);
+            if (!/^https?:$/.test(parsed.protocol))
+                throw new Error('not http');
+        }
+        catch {
+            return usage(`"${url}" is not an http(s) URL.`);
+        }
+    }
+    const siteUrl = (0, api_base_1.siteFromEnv)(env);
+    const platformHost = ((0, args_1.flag)(p, 'platform-host') ??
+        (siteUrl ? new URL(siteUrl).host : undefined) ??
+        extended_1.DEFAULT_PLATFORM_HOST).toLowerCase();
+    const fetchImpl = deps.fetchImpl ?? fetch;
+    const addressSets = [];
+    let roots;
+    const site = (0, args_1.flag)(p, 'site');
+    // The organization's own checks (reader-auth-atlas) need an org and a token;
+    // `--site` needs both anyway.
+    const orgFlag = (0, args_1.flag)(p, 'org');
+    const orgRef = site || orgFlag ? (0, config_1.setting)('org', orgFlag, env, (0, config_1.readConfig)(env)) : undefined;
+    const apiHttp = () => {
+        const baseUrl = (0, api_base_1.apiBaseFromEnv)(env);
+        return (deps.http ??
+            new http_1.Http({
+                baseUrl,
+                token: (0, credentials_1.resolveToken)(baseUrl, env, deps.flagToken),
+                userAgent: deps.userAgent ?? '@forumulate/cli',
+                fetch: fetchImpl,
+            }));
+    };
+    if (site) {
+        const org = orgRef;
+        if (!org)
+            return usage('--site needs --org <slug|id> (or FORUMULATE_ORG, or `config set org`).');
+        const http = apiHttp();
+        try {
+            const resolved = await (0, site_1.resolveSite)(http, org, site);
+            for (const problem of resolved.problems)
+                deps.err(`note: ${problem}`);
+            if (!url)
+                url = resolved.url;
+            if (!url) {
+                deps.err(`Site ${site} has no space with a public address to check.`);
+                return exit_codes_1.EXIT.NOT_FOUND;
+            }
+            roots = resolved.spaces.flatMap((s) => s.endpoints.map((e) => e.url.replace(/\/+$/, '')));
+            const get = (0, probe_1.makeGet)({ fetchImpl, timeoutMs: timeout });
+            addressSets.push(...(await (0, site_1.addressSetsFor)(get, resolved)));
+        }
+        catch (e) {
+            const err = e;
+            deps.err(`Could not read site ${site}: ${err.message ?? String(e)}`);
+            return (0, exit_codes_1.exitCodeForError)(e);
+        }
+    }
+    if (!url)
+        return usage('Give a reader URL, or --site <slug|id>.');
+    const also = (0, args_1.all)(p, 'also');
+    if (also.length)
+        addressSets.push({ urls: [url, ...also] });
+    const report = await (0, doctor_1.runDoctor)({
+        url,
+        apiBase: (0, args_1.flag)(p, 'api-base') ?? siteUrl,
+        fetchImpl,
+        only: only.length ? only : undefined,
+        platformHost,
+        // The platform the copies are compared against: the configured API when
+        // one is, else the API of whichever host --platform-host named.
+        platformApi: env.FORUMULATE_API_URL?.trim() || env.FORUMULATE_URL?.trim()
+            ? (0, api_base_1.apiBaseFromEnv)(env)
+            : `https://${platformHost}/api`,
+        addressSets: addressSets.length ? addressSets : undefined,
+        roots,
+        limit,
+        timeoutMs: timeout,
+    });
+    // FMU-X01: the organization's reader sign-in, from the API's own recommendation.
+    if (orgRef && (!only.length || only.includes(reader_auth_atlas_1.READER_AUTH_ATLAS_CHECK))) {
+        const c = await (0, reader_auth_atlas_1.readerAuthAtlasCheck)(apiHttp(), orgRef);
+        report.checks.push(c);
+        if (c.state === 'pass')
+            report.passed += 1;
+        else if (c.state === 'fail')
+            report.failed += 1;
+        else
+            report.warnings += 1;
+    }
+    if (json)
+        deps.out(JSON.stringify(report, null, 2));
+    else
+        printReport(report, deps.out);
+    return report.failed > 0 ? exit_codes_1.EXIT.FAILED : exit_codes_1.EXIT.OK;
+}
+function printReport(report, out) {
+    out(`Checking ${report.url}`);
+    for (const c of report.checks) {
+        const mark = c.state === 'pass' ? 'PASS' : c.state === 'fail' ? 'FAIL' : 'SKIP';
+        out(`  ${mark}  ${c.id}  ${c.title}`);
+        // A detail may be several lines (robots-params prints a robots.txt block).
+        for (const line of c.detail.split('\n'))
+            out(`        ${line}`);
+        if (c.state === 'fail' && c.matters)
+            out(`        why it matters: ${c.matters}`);
+    }
+    out(`${report.passed} passed, ${report.failed} failed, ${report.warnings} skipped`);
+}
+//# sourceMappingURL=command.js.map

package/dist/doctor/command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"command.js","sourceRoot":"","sources":["../../src/doctor/command.ts"],"names":[],"mappings":";;;AA2GA,4CA4HC;AAED,kCAUC;AAnPD,sCAAwD;AACxD,sCAAmC;AACnC,0CAA0D;AAC1D,+CAAyD;AACzD,gDAA8C;AAC9C,8CAAwE;AACxE,sCAAyE;AACzE,yCAA6F;AAC7F,iCAAqD;AACrD,mCAAkC;AAElC,2DAAoF;AAEvE,QAAA,gBAAgB,GAAsB;IACjD,GAAG,uBAAc;IACjB,GAAG,6BAAkB;IACrB,2CAAuB;CACxB,CAAC;AAEW,QAAA,WAAW,GAAG;;;;;;;;;;;;;;;;;;;;;yBAqBF,gCAAqB;yDACW,+BAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiD3E,4BAAe;2DAC0C,CAAC;AAa5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;AAEhD,4CAA4C;AACrC,KAAK,UAAU,gBAAgB,CAAC,IAAc,EAAE,IAAuB;IAC5E,MAAM,CAAC,GAAG,IAAA,gBAAS,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpC,IAAI,IAAA,UAAG,EAAC,CAAC,EAAE,MAAM,CAAC,IAAI,IAAA,UAAG,EAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,mBAAW,CAAC,CAAC;QACtB,OAAO,iBAAI,CAAC,EAAE,CAAC;IACjB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACpC,MAAM,IAAI,GAAG,IAAA,UAAG,EAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,oFAAoF,CAAC,CAAC;QAC/F,OAAO,iBAAI,CAAC,KAAK,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,IAAA,UAAG,EAAC,CAAC,EAAE,MAAM,CAAC;SACxB,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,wBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,IAAI,OAAO,CAAC,MAAM;QAChB,OAAO,KAAK,CACV,wBAAwB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,wBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACrF,CAAC;IAEJ,MAAM,KAAK,GAAG,IAAA,WAAI,EAAC,CAAC,EAAE,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAA,WAAI,EAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IACpF,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;QAChE,OAAO,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,IAAA,WAAI,EAAC,CAAC,EAAE,SAAS,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAA,WAAI,EAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC1F,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC;QACtE,OAAO,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAErE,IAAI,GAAG,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,0BAA0B,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjG,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC,IAAI,GAAG,0BAA0B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,sBAAW,EAAC,GAAG,CAAC,CAAC;IACjC,MAAM,YAAY,GAAG,CACnB,IAAA,WAAI,EAAC,CAAC,EAAE,eAAe,CAAC;QACxB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7C,gCAAqB,CACtB,CAAC,WAAW,EAAE,CAAC;IAChB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,IAAI,KAA2B,CAAC;IAEhC,MAAM,IAAI,GAAG,IAAA,WAAI,EAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7B,6EAA6E;IAC7E,8BAA8B;IAC9B,MAAM,OAAO,GAAG,IAAA,WAAI,EAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,IAAA,gBAAO,EAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,IAAA,mBAAU,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3F,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,MAAM,OAAO,GAAG,IAAA,yBAAc,EAAC,GAAG,CAAC,CAAC;QACpC,OAAO,CACL,IAAI,CAAC,IAAI;YACT,IAAI,WAAI,CAAC;gBACP,OAAO;gBACP,KAAK,EAAE,IAAA,0BAAY,EAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC;gBACjD,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,iBAAiB;gBAC9C,KAAK,EAAE,SAAS;aACjB,CAAC,CACH,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,MAAM,CAAC;QACnB,IAAI,CAAC,GAAG;YACN,OAAO,KAAK,CAAC,wEAAwE,CAAC,CAAC;QACzF,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAA,kBAAW,EAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YACpD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,QAAQ;gBAAE,IAAI,CAAC,GAAG,CAAC,SAAS,OAAO,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,GAAG;gBAAE,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,+CAA+C,CAAC,CAAC;gBACtE,OAAO,iBAAI,CAAC,SAAS,CAAC;YACxB,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAC1F,MAAM,GAAG,GAAG,IAAA,eAAO,EAAC,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;YACvD,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAA,qBAAc,EAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAA0C,CAAC;YACvD,IAAI,CAAC,GAAG,CAAC,uBAAuB,IAAI,KAAK,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO,IAAA,6BAAgB,EAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,IAAA,UAAG,EAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC5B,IAAI,IAAI,CAAC,MAAM;QAAE,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAS,EAAC;QAC7B,GAAG;QACH,OAAO,EAAE,IAAA,WAAI,EAAC,CAAC,EAAE,UAAU,CAAC,IAAI,OAAO;QACvC,SAAS;QACT,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QACpC,YAAY;QACZ,wEAAwE;QACxE,gEAAgE;QAChE,WAAW,EACT,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE;YAC1D,CAAC,CAAC,IAAA,yBAAc,EAAC,GAAG,CAAC;YACrB,CAAC,CAAC,WAAW,YAAY,MAAM;QACnC,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;QACzD,KAAK;QACL,KAAK;QACL,SAAS,EAAE,OAAO;KACnB,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,2CAAuB,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,CAAC,GAAG,MAAM,IAAA,wCAAoB,EAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM;YAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;aACtC,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM;YAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;;YAC3C,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,IAAI,IAAI;QAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;;QAC/C,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAI,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAI,CAAC,EAAE,CAAC;AACnD,CAAC;AAED,SAAgB,WAAW,CAAC,MAAoB,EAAE,GAAwB;IACxE,GAAG,CAAC,YAAY,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAChF,GAAG,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACtC,2EAA2E;QAC3E,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,CAAC,CAAC,OAAO;YAAE,GAAG,CAAC,2BAA2B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,QAAQ,UAAU,CAAC,CAAC;AACtF,CAAC"}

package/dist/doctor/crawl.d.ts

@@ -0,0 +1,49 @@
+import type { Check } from '../conformance';
+import { type Get, type Probe } from './probe';
+export interface CrawlResult {
+    check: Check;
+    /** Every listed URL that was fetched, with its answer. */
+    pages: Probe[];
+}
+/** `Sitemap:` lines of robots.txt, else the conventional address. */
+export declare function sitemapsFromRobots(robots: string, origin: string): string[];
+/**
+ * Read sitemaps (following a sitemap index), returning the URL lists per
+ * document, so the sample can draw from every space rather than exhaust the
+ * first one: a site with a 900-page reference and a 4-page legal space would
+ * otherwise never have its legal pages looked at.
+ */
+export declare function readSitemaps(get: Get, start: string[]): Promise<{
+    lists: {
+        sitemap: string;
+        urls: string[];
+    }[];
+    problems: string[];
+}>;
+/** Up to `limit` URLs, round-robin across the sitemaps so every space is represented. */
+export declare function sample(lists: {
+    urls: string[];
+}[], limit: number): string[];
+export declare function sitemapCrawl(get: Get, origin: string, robots: string | undefined, limit: number): Promise<CrawlResult>;
+/**
+ * The space a URL belongs to, for grouping: its first path segment, which is
+ * where a path-mounted space lives (`/docs`, `/legal`). Pages at the root of a
+ * site domain group together under `/`. Coarse, and that is fine — a
+ * difference between two spaces is a choice; a difference inside one is what
+ * this is looking for.
+ */
+export declare function groupOf(url: string): string;
+/** `public` (a shared cache may keep it) or `private` (it may not), by the directives that decide it. */
+export declare function cacheClass(cc: string | null): 'public' | 'private' | 'none';
+/**
+ * `cache-consistency`: the pages of one space are cached the same way.
+ *
+ * Every legal document answered `private, no-cache, no-store` to a reader with
+ * no cookie at all, while the legal index beside it was `public, s-maxage=10`
+ * (FMU-B05). Neither header is wrong on its own; the difference is the defect —
+ * either something is gated and should say so, or a code path forgot the
+ * public header and every reader pays a full origin render. A page that sends
+ * a reason header is taken at its word.
+ */
+export declare function cacheConsistency(pages: Probe[]): Check;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/doctor/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/doctor/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAQL,KAAK,GAAG,EACR,KAAK,KAAK,EACX,MAAM,SAAS,CAAC;AAmBjB,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,0DAA0D;IAC1D,KAAK,EAAE,KAAK,EAAE,CAAC;CAChB;AAED,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAG3E;AAQD;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,GAAG,EACR,KAAK,EAAE,MAAM,EAAE,GACd,OAAO,CAAC;IAAE,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAkB/E;AAED,yFAAyF;AACzF,wBAAgB,MAAM,CAAC,KAAK,EAAE;IAAE,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAkB3E;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,GAAG,EACR,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,WAAW,CAAC,CAuDtB;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAY3C;AAED,yGAAyG;AACzG,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAK3E;AAKD;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,KAAK,CA0CtD"}

package/dist/doctor/crawl.js

@@ -0,0 +1,210 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sitemapsFromRobots = sitemapsFromRobots;
+exports.readSitemaps = readSitemaps;
+exports.sample = sample;
+exports.sitemapCrawl = sitemapCrawl;
+exports.groupOf = groupOf;
+exports.cacheClass = cacheClass;
+exports.cacheConsistency = cacheConsistency;
+const probe_1 = require("./probe");
+/**
+ * `sitemap-crawl`: every address the site hands a crawler, fetched.
+ *
+ * The sitemap is the list a search engine trusts most, so an entry that
+ * redirects, 404s, names a different canonical or sits on another host is a
+ * page the site is asking to have indexed and then refusing to be. A space root
+ * that 307s to its first page was listed as itself (FMU-B06); a platform copy
+ * that 308s elsewhere named itself canonical (FMU-B01). Each is invisible from
+ * the page a person happens to open, and obvious once every listed URL is asked.
+ *
+ * The pages fetched here are also the SAMPLE the other site-wide checks judge
+ * (cache headers, descriptions), so the crawl is done once.
+ */
+/** How many sitemap documents (index included) one run will read. */
+const SITEMAP_DOC_LIMIT = 25;
+/** `Sitemap:` lines of robots.txt, else the conventional address. */
+function sitemapsFromRobots(robots, origin) {
+    const named = [...robots.matchAll(/^\s*sitemap:\s*(\S+)/gim)].map((m) => m[1]);
+    return named.length ? named : [`${origin}/sitemap.xml`];
+}
+function locs(xml) {
+    return [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => m[1].replace(/&amp;/g, '&'));
+}
+/**
+ * Read sitemaps (following a sitemap index), returning the URL lists per
+ * document, so the sample can draw from every space rather than exhaust the
+ * first one: a site with a 900-page reference and a 4-page legal space would
+ * otherwise never have its legal pages looked at.
+ */
+async function readSitemaps(get, start) {
+    const lists = [];
+    const problems = [];
+    const queue = [...new Set(start)];
+    const seen = new Set();
+    while (queue.length && seen.size < SITEMAP_DOC_LIMIT) {
+        const url = queue.shift();
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        const p = await get(url, { accept: 'application/xml,text/xml' });
+        if (!p.ok || p.status !== 200) {
+            problems.push(`${url} → ${(0, probe_1.describeProbe)(p)}`);
+            continue;
+        }
+        if (/<sitemapindex\b/i.test(p.text))
+            queue.push(...locs(p.text));
+        else
+            lists.push({ sitemap: url, urls: locs(p.text) });
+    }
+    return { lists, problems };
+}
+/** Up to `limit` URLs, round-robin across the sitemaps so every space is represented. */
+function sample(lists, limit) {
+    const out = [];
+    const seen = new Set();
+    for (let i = 0; out.length < limit; i++) {
+        let any = false;
+        for (const l of lists) {
+            const u = l.urls[i];
+            if (u === undefined)
+                continue;
+            any = true;
+            if (!seen.has(u)) {
+                seen.add(u);
+                out.push(u);
+                if (out.length >= limit)
+                    break;
+            }
+        }
+        if (!any)
+            break;
+    }
+    return out;
+}
+async function sitemapCrawl(get, origin, robots, limit) {
+    const title = 'Every URL in the sitemap answers 200, as itself';
+    const matters = 'A sitemap entry that redirects, 404s or names another canonical is a page the site asks to have indexed and then disowns; search engines lower their trust in the whole list.';
+    const { lists, problems } = await readSitemaps(get, sitemapsFromRobots(robots ?? '', origin));
+    if (lists.length === 0) {
+        return {
+            check: (0, probe_1.check)('sitemap-crawl', title, 'fail', problems.length ? `No sitemap could be read: ${(0, probe_1.some)(problems)}` : 'No sitemap found.', matters),
+            pages: [],
+        };
+    }
+    const total = lists.reduce((n, l) => n + l.urls.length, 0);
+    const urls = sample(lists, limit);
+    const siteHost = (0, probe_1.hostOf)(origin);
+    const pages = await (0, probe_1.pool)(urls, 6, (u) => get(u, { accept: 'text/html' }));
+    const bad = [...problems];
+    for (const p of pages) {
+        if ((0, probe_1.hostOf)(p.url) !== siteHost) {
+            bad.push(`${p.url} is on another host than the sitemap (${siteHost})`);
+            continue;
+        }
+        if (!p.ok) {
+            bad.push(`${p.url} → ${p.error}`);
+            continue;
+        }
+        if (p.status >= 300 && p.status < 400) {
+            bad.push(`${p.url} → ${p.status} ${p.headers.get('location') ?? ''} (list the final URL)`);
+            continue;
+        }
+        if (p.status !== 200) {
+            bad.push(`${p.url} → ${p.status}`);
+            continue;
+        }
+        const canonical = (0, probe_1.canonicalOf)(p.text, p.url);
+        if (canonical && !(0, probe_1.sameUrl)(canonical, p.url))
+            bad.push(`${p.url} names canonical ${canonical}`);
+    }
+    const scope = `${pages.length} of ${total} listed URL${total === 1 ? '' : 's'} across ${lists.length} sitemap${lists.length === 1 ? '' : 's'}`;
+    return {
+        check: (0, probe_1.check)('sitemap-crawl', title, bad.length ? 'fail' : 'pass', bad.length
+            ? `${bad.length} problem(s) in ${scope}: ${(0, probe_1.some)(bad)}`
+            : `${scope}: all 200, all self-canonical.`, bad.length ? matters : ''),
+        pages,
+    };
+}
+/**
+ * The space a URL belongs to, for grouping: its first path segment, which is
+ * where a path-mounted space lives (`/docs`, `/legal`). Pages at the root of a
+ * site domain group together under `/`. Coarse, and that is fine — a
+ * difference between two spaces is a choice; a difference inside one is what
+ * this is looking for.
+ */
+function groupOf(url) {
+    try {
+        const u = new URL(url);
+        const segs = u.pathname.split('/').filter(Boolean);
+        // A short reader prefix followed by owner and space (`/p/<org>/<space>/…`)
+        // is how a multi-tenant host addresses a space; grouping by `/p` alone
+        // would compare every tenant's pages with every other's.
+        if (segs.length >= 4 && segs[0].length <= 3)
+            return `${u.host}/${segs.slice(0, 3).join('/')}`;
+        return `${u.host}/${segs[0] ?? ''}`;
+    }
+    catch {
+        return url;
+    }
+}
+/** `public` (a shared cache may keep it) or `private` (it may not), by the directives that decide it. */
+function cacheClass(cc) {
+    if (!cc)
+        return 'none';
+    if (/\b(?:private|no-store)\b/i.test(cc))
+        return 'private';
+    if (/\b(?:public|s-maxage)\b/i.test(cc))
+        return 'public';
+    return 'private';
+}
+/** Headers a site may send to say a page is uncached on purpose (and why). */
+const REASON_HEADERS = ['x-forumulate-cache-reason', 'x-cache-reason', 'x-cache-control-reason'];
+/**
+ * `cache-consistency`: the pages of one space are cached the same way.
+ *
+ * Every legal document answered `private, no-cache, no-store` to a reader with
+ * no cookie at all, while the legal index beside it was `public, s-maxage=10`
+ * (FMU-B05). Neither header is wrong on its own; the difference is the defect —
+ * either something is gated and should say so, or a code path forgot the
+ * public header and every reader pays a full origin render. A page that sends
+ * a reason header is taken at its word.
+ */
+function cacheConsistency(pages) {
+    const title = 'Pages within a space are cached alike';
+    const groups = new Map();
+    for (const p of pages) {
+        if (!p.ok || p.status !== 200)
+            continue;
+        if (REASON_HEADERS.some((h) => p.headers.get(h)))
+            continue;
+        const cc = p.headers.get('cache-control');
+        const g = groupOf(p.url);
+        groups.set(g, [
+            ...(groups.get(g) ?? []),
+            { url: p.url, cls: cacheClass(cc), cc: cc ?? '(none)' },
+        ]);
+    }
+    if (groups.size === 0)
+        return (0, probe_1.check)('cache-consistency', title, 'skip', 'No sampled pages answered 200.', '');
+    const odd = [];
+    for (const [g, rows] of groups) {
+        const classes = [...new Set(rows.map((r) => r.cls))];
+        if (classes.length < 2)
+            continue;
+        // Every side of the split is named. Which side is the bug is the site's
+        // call (usually the private one — a published page has no reason to be),
+        // and a "majority" of three legal documents is evidence of nothing.
+        const sides = classes.map((cls) => {
+            const of = rows.filter((r) => r.cls === cls);
+            return `${of.length} ${cls} ("${of[0].cc}": ${(0, probe_1.some)(of.map((r) => r.url), 3)})`;
+        });
+        odd.push(`${g}: ${sides.join(' vs ')}`);
+    }
+    return (0, probe_1.check)('cache-consistency', title, odd.length ? 'fail' : 'pass', odd.length
+        ? (0, probe_1.some)(odd)
+        : `${[...groups.values()].reduce((n, r) => n + r.length, 0)} pages in ${groups.size} space(s), one rule each.`, odd.length
+        ? 'A published page answering no-store to an anonymous reader is rendered at the origin for every visit. If it is gated on purpose, say so with an x-forumulate-cache-reason header.'
+        : '');
+}
+//# sourceMappingURL=crawl.js.map

package/dist/doctor/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/doctor/crawl.ts"],"names":[],"mappings":";;AAqCA,gDAGC;AAcD,oCAqBC;AAGD,wBAkBC;AAED,oCA4DC;AASD,0BAYC;AAGD,gCAKC;AAeD,4CA0CC;AAnPD,mCAUiB;AAEjB;;;;;;;;;;;;GAYG;AAEH,qEAAqE;AACrE,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAQ7B,qEAAqE;AACrE,SAAgB,kBAAkB,CAAC,MAAc,EAAE,MAAc;IAC/D,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IAChF,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClE,CAAC,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,YAAY,CAChC,GAAQ,EACR,KAAe;IAEf,MAAM,KAAK,GAA0C,EAAE,CAAC;IACxD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,GAAG,iBAAiB,EAAE,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC9B,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9C,SAAS;QACX,CAAC;QACD,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;;YAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,yFAAyF;AACzF,SAAgB,MAAM,CAAC,KAA2B,EAAE,KAAa;IAC/D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,KAAK,SAAS;gBAAE,SAAS;YAC9B,GAAG,GAAG,IAAI,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACZ,IAAI,GAAG,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAM;YACjC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG;YAAE,MAAM;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAEM,KAAK,UAAU,YAAY,CAChC,GAAQ,EACR,MAAc,EACd,MAA0B,EAC1B,KAAa;IAEb,MAAM,KAAK,GAAG,iDAAiD,CAAC;IAChE,MAAM,OAAO,GACX,+KAA+K,CAAC;IAClL,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,kBAAkB,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,KAAK,EAAE,IAAA,aAAK,EACV,eAAe,EACf,KAAK,EACL,MAAM,EACN,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B,IAAA,YAAI,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EACrF,OAAO,CACR;YACD,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAA,cAAM,EAAC,MAAM,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,IAAA,YAAI,EAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAa,CAAC,GAAG,QAAQ,CAAC,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAA,cAAM,EAAC,CAAC,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,yCAAyC,QAAQ,GAAG,CAAC,CAAC;YACvE,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAClC,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;YAC3F,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,IAAA,mBAAW,EAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,SAAS,IAAI,CAAC,IAAA,eAAO,EAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,oBAAoB,SAAS,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,MAAM,OAAO,KAAK,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,MAAM,WAAW,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC/I,OAAO;QACL,KAAK,EAAE,IAAA,aAAK,EACV,eAAe,EACf,KAAK,EACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAC5B,GAAG,CAAC,MAAM;YACR,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,kBAAkB,KAAK,KAAK,IAAA,YAAI,EAAC,GAAG,CAAC,EAAE;YACtD,CAAC,CAAC,GAAG,KAAK,gCAAgC,EAC5C,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC1B;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,OAAO,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnD,2EAA2E;QAC3E,uEAAuE;QACvE,yDAAyD;QACzD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/F,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,yGAAyG;AACzG,SAAgB,UAAU,CAAC,EAAiB;IAC1C,IAAI,CAAC,EAAE;QAAE,OAAO,MAAM,CAAC;IACvB,IAAI,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3D,IAAI,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,QAAQ,CAAC;IACzD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,MAAM,cAAc,GAAG,CAAC,2BAA2B,EAAE,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;AAEjG;;;;;;;;;GASG;AACH,SAAgB,gBAAgB,CAAC,KAAc;IAC7C,MAAM,KAAK,GAAG,uCAAuC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsD,CAAC;IAC7E,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG;YAAE,SAAS;QACxC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QAC3D,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;YACZ,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACxB,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,QAAQ,EAAE;SACxD,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QACnB,OAAO,IAAA,aAAK,EAAC,mBAAmB,EAAE,KAAK,EAAE,MAAM,EAAE,gCAAgC,EAAE,EAAE,CAAC,CAAC;IACzF,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACjC,wEAAwE;QACxE,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YAC7C,OAAO,GAAG,EAAE,CAAC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,CAAC,CAAE,CAAC,EAAE,MAAM,IAAA,YAAI,EACjD,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EACpB,CAAC,CACF,GAAG,CAAC;QACP,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAA,aAAK,EACV,mBAAmB,EACnB,KAAK,EACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAC5B,GAAG,CAAC,MAAM;QACR,CAAC,CAAC,IAAA,YAAI,EAAC,GAAG,CAAC;QACX,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,aAAa,MAAM,CAAC,IAAI,2BAA2B,EAChH,GAAG,CAAC,MAAM;QACR,CAAC,CAAC,mLAAmL;QACrL,CAAC,CAAC,EAAE,CACP,CAAC;AACJ,CAAC"}

package/dist/doctor/extended.d.ts

@@ -0,0 +1,52 @@
+import type { Check } from '../conformance';
+import { type AddressSet } from './address-checks';
+import { type Get } from './probe';
+/**
+ * The site-wide checks added for FMU-D14, run after the single-page ones.
+ *
+ * Every one of them is a defect an integrator found on a live site after
+ * readers did, and every one is judged from the outside, by HTTP alone, so the
+ * report is the same whether it is pointed at our platform, a staging mount or
+ * any other docs host. The only thing a check knows about "the platform" is
+ * its hostname, which is an input (`--platform-host`, else the host of
+ * FORUMULATE_URL, else forumulate.com) — never a constant in the logic.
+ */
+/** Every check id this file can produce, in report order. `--only` validates against it. */
+export declare const EXTENDED_CHECK_IDS: readonly ["canonical-both-addresses", "sitemap-crawl", "cache-consistency", "root-redirect", "meta-description", "llms-txt", "favicon", "consent-tag", "white-label-leaks", "robots-params"];
+export declare const DEFAULT_PLATFORM_HOST = "forumulate.com";
+/** URLs one run fetches from the sitemaps; enough to see a pattern, few enough for CI. */
+export declare const DEFAULT_SAMPLE_LIMIT = 60;
+export interface ExtendedInput {
+    /** The page judged: where the asked-for address landed after redirects. */
+    url: string;
+    /** The address as given, before any redirect. */
+    requestedUrl?: string;
+    /** The page's HTML as an anonymous reader got it ('' when it did not answer). */
+    html: string;
+    /** robots.txt, when it answered 200. */
+    robots?: string;
+    get: Get;
+    platformHost: string;
+    /** The platform's API base (`https://forumulate.com/api`), for the primary-address lookup. */
+    platformApi?: string;
+    /** Other addresses of this page/space, from `--also` or `--site`. */
+    addressSets?: AddressSet[];
+    /** Space roots to check, from `--site`; derived from the URL otherwise. */
+    roots?: string[];
+    limit?: number;
+    /** Run only these ids (undefined: all). */
+    only?: ReadonlySet<string>;
+}
+/** The roots a URL's space is served at: the mount (first segment), or the platform copy's space root. */
+export declare function deriveRoots(url: string, platformHost: string): string[];
+/**
+ * The customer's address of a platform-hosted page, asked of the platform.
+ *
+ * Only for a URL on the platform host in the platform-copy shape: the public
+ * primary-domain lookup answers where the space's readers are sent, which is
+ * what both copies must name as canonical. Any failure is "unknown", and the
+ * check then compares the copies it has with each other.
+ */
+export declare function discoverPrimary(get: Get, url: string, platformHost: string, platformApi: string | undefined): Promise<AddressSet | undefined>;
+export declare function extendedChecks(input: ExtendedInput): Promise<Check[]>;
+//# sourceMappingURL=extended.d.ts.map

package/dist/doctor/extended.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extended.d.ts","sourceRoot":"","sources":["../../src/doctor/extended.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAiB,KAAK,GAAG,EAAc,MAAM,SAAS,CAAC;AAG9D;;;;;;;;;GASG;AAEH,4FAA4F;AAC5F,eAAO,MAAM,kBAAkB,8LAWrB,CAAC;AAEX,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AACtD,0FAA0F;AAC1F,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,MAAM,WAAW,aAAa;IAC5B,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,iDAAiD;IACjD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,GAAG,CAAC;IACT,YAAY,EAAE,MAAM,CAAC;IACrB,8FAA8F;IAC9F,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,IAAI,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC5B;AASD,0GAA0G;AAC1G,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,EAAE,CASvE;AAED;;;;;;;GAOG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,MAAM,EACX,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,GAAG,SAAS,GAC9B,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAuBjC;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAiF3E"}