katman-mcp 0.1.0

package/dist/tools/crawl/audit/analyze.js

@@ -0,0 +1,719 @@
+import { crawlerByToken } from "../../../crawlers.js";
+import { CSR_ACCEPT, JS_RENDER_NOTE, LOVABLE_CAVEAT, cloudflareFix, csrFixFor, genericCdnFix } from "../common/advice.js";
+import { classifyPath, isContentKind } from "../common/routes.js";
+import { collapse, containment, listSome, pct } from "../common/text.js";
+import { safeUrl, sameUrlExact, shortUrl, urlKey } from "../common/urls.js";
+import { prefixCollateral } from "./robots.js";
+function F(id, severity, gate, prompt, title, detail, extra = {}) {
+    return { id, title, severity, gate, prompt, detail, ...extra };
+}
+/** AI search + user bots: if the CDN refuses them, nothing else on the page matters. */
+const GATE_BOTS = new Set(["OAI-SearchBot", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User"]);
+// ---------------------------------------------------------------------------------------------------------------
+// Page predicates (shared with the report)
+export function htmlOk(p) {
+    return p.status === 200 && p.isHtml && !!p.html;
+}
+export function indexable(p) {
+    return htmlOk(p) && !p.noindex;
+}
+/** Pages that count for the read/quote gates: public content pages that are pages (not redirects/assets). */
+/** The page names another URL as its canonical (a declared duplicate, e.g. /start?plan=pro → /start). */
+export function canonicalizedAway(p) {
+    return !!p.html?.canonical && canonicalSelf(p) === false && !p.inSitemap && p.source !== "home";
+}
+/** Pages that count for the read/quote gates: public content pages that are pages (not redirects, assets or declared duplicates). */
+export function gatePopulation(ctx) {
+    return ctx.pages.filter((p) => isContentKind(p.kind) &&
+        (p.status === 200 || p.inSitemap || p.source === "input" || p.source === "home") &&
+        (p.isHtml || p.status !== 200) &&
+        !canonicalizedAway(p));
+}
+export function blockedGateBots(ctx) {
+    return (ctx.bots?.probes ?? []).filter((b) => b.blocked && GATE_BOTS.has(b.crawler.token)).map((b) => b.crawler.token);
+}
+/** Words in the text right after the H1 (the answer block, whatever element it is in). */
+function wordsAfterH1(p) {
+    return (p.html?.textAfterH1 ?? "").split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
+}
+/** P2 acceptance: with JavaScript off the H1 and the first paragraph (answer text) are readable. */
+export function jsOffOk(p) {
+    if (!p.html || !p.html.h1Text)
+        return false;
+    return p.html.wordsJsOff >= 100 || !!p.html.firstParagraph || wordsAfterH1(p) >= 25;
+}
+/** Strong evidence that the page's text is drawn by JavaScript (not just a short page). */
+export function csrSuspected(p) {
+    const h = p.html;
+    if (!h)
+        return false;
+    if (h.rootShell)
+        return true;
+    if (h.wordsJsOff < 50)
+        return true;
+    return h.wordsJsOff < 100 && !h.h1Text && (p.bytes > 15_000 || h.scriptCount >= 3);
+}
+export function readFailures(ctx, p) {
+    const r = [];
+    if (p.status !== 200)
+        r.push(p.status ? `HTTP ${p.status}` : "no response");
+    else if (!p.isHtml)
+        r.push("not HTML");
+    if (p.noindex)
+        r.push("noindex");
+    if (p.robotsBlocked.length)
+        r.push(`robots.txt blocks ${p.robotsBlocked.join("/")}`);
+    if (p.html && !jsOffOk(p))
+        r.push(!p.html.h1Text ? "no H1 without JavaScript" : "no answer text after the H1 without JavaScript");
+    if (blockedGateBots(ctx).length)
+        r.push("AI bots refused by the CDN/firewall");
+    return r;
+}
+export function uniqueness(ctx) {
+    const title = new Map();
+    const desc = new Map();
+    for (const p of gatePopulation(ctx)) {
+        if (!indexable(p))
+            continue;
+        const t = norm(p.html.title);
+        const d = norm(p.html.description);
+        if (t)
+            title.set(t, [...(title.get(t) ?? []), p]);
+        if (d)
+            desc.set(d, [...(desc.get(d) ?? []), p]);
+    }
+    return { title, desc };
+}
+function norm(s) {
+    return collapse(s ?? "").toLowerCase();
+}
+export function titleUnique(u, p) {
+    const t = norm(p.html?.title);
+    return !!t && (u.title.get(t)?.length ?? 0) <= 1;
+}
+export function descUnique(u, p) {
+    const d = norm(p.html?.description);
+    return !!d && (u.desc.get(d)?.length ?? 0) <= 1;
+}
+export function canonicalSelf(p) {
+    if (!p.html?.canonical)
+        return undefined;
+    return sameUrlExact(p.html.canonical, p.url);
+}
+export function quoteFailures(ctx, p, u) {
+    const r = [];
+    if (!p.html)
+        return ["no HTML"];
+    if (!p.html.firstParagraph && wordsAfterH1(p) < 25)
+        r.push("no first paragraph (≥40 chars) in the HTML");
+    if (!p.html.title)
+        r.push("no title");
+    else if (!titleUnique(u, p))
+        r.push("duplicate title");
+    if (!p.html.description)
+        r.push("no meta description");
+    else if (!descUnique(u, p))
+        r.push("duplicate description");
+    if (p.html.jsonLdErrors)
+        r.push("JSON-LD does not parse");
+    if (p.html.nosnippet)
+        r.push("nosnippet");
+    if (p === ctx.home && ctx.sentence) {
+        const cp = containment(ctx.sentence, `${p.html.firstParagraph ?? ""} ${p.html.description ?? ""}`);
+        if (cp < 0.5)
+            r.push(`one sentence missing (${pct(Math.round(cp * 100), 100)} overlap)`);
+    }
+    return r;
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Gates
+export function computeGates(ctx) {
+    const pop = gatePopulation(ctx);
+    const u = uniqueness(ctx);
+    const readNotes = new Map();
+    const quoteNotes = new Map();
+    let readPass = 0;
+    let quotePass = 0;
+    for (const p of pop) {
+        const rf = readFailures(ctx, p);
+        if (!rf.length)
+            readPass++;
+        for (const r of rf)
+            readNotes.set(r.replace(/^\d+ words/, "<100 words"), (readNotes.get(r.replace(/^\d+ words/, "<100 words")) ?? 0) + 1);
+        const qf = rf.length ? ["fails read"] : quoteFailures(ctx, p, u);
+        if (!qf.length)
+            quotePass++;
+        for (const q of qf)
+            quoteNotes.set(q.replace(/\(\d+% overlap\)/, "").trim(), (quoteNotes.get(q.replace(/\(\d+% overlap\)/, "").trim()) ?? 0) + 1);
+    }
+    const top = (m) => [...m.entries()].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([k, n]) => `${n} page${n === 1 ? "" : "s"}: ${k}`);
+    const readRule = "Rule: HTTP 200 HTML, no noindex, not blocked in robots.txt for Googlebot/OAI-SearchBot, an H1 plus answer text in the raw HTML without JavaScript (a first paragraph, ≥25 words right after the H1, or ≥100 words), and AI search/user bots not refused by the CDN.";
+    const quoteRule = "Rule: passes read + a first paragraph (≥40 chars) in the HTML + unique title + unique meta description + JSON-LD parses + no nosnippet; homepage also carries the research one sentence (≥50% word overlap) when research.json has it.";
+    const fitRule = "Rule: each research situation has a page that returns 200, is indexable and has its H1 in the raw HTML.";
+    const fitNotes = [fitRule];
+    let fitPass = 0;
+    if (!ctx.research)
+        fitNotes.push("No .katman/research.json yet: run katman_research to list the customer situations.");
+    else if (!ctx.situations.length)
+        fitNotes.push("research.json has no situations yet.");
+    for (const s of ctx.situations) {
+        if (s.live && s.h1)
+            fitPass++;
+        else
+            fitNotes.push(`"${s.sentence.slice(0, 60)}": ${!s.url ? "no page planned (no url)" : s.status ? `${s.url} → HTTP ${s.status}${s.noindex ? " noindex" : ""}${s.live && !s.h1 ? " (no H1 without JS)" : ""}` : `${s.url} not checked`}`);
+    }
+    return {
+        read: { pass: readPass, total: pop.length, notes: [readRule, ...top(readNotes)] },
+        quote: { pass: quotePass, total: pop.length, notes: [quoteRule, ...top(quoteNotes)] },
+        fit: { pass: fitPass, total: ctx.situations.length, notes: fitNotes.slice(0, 14) },
+    };
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Findings
+export function buildFindings(ctx) {
+    const out = [];
+    siteFindings(ctx, out);
+    robotsFindings(ctx, out);
+    sitemapFindings(ctx, out);
+    botFindings(ctx, out);
+    pageFindings(ctx, out);
+    linkFindings(ctx, out);
+    researchFindings(ctx, out);
+    return out;
+}
+function u(ctx, url) {
+    return shortUrl(url, ctx.host);
+}
+function siteFindings(ctx, out) {
+    const home = ctx.home;
+    if (!home || home.status === 0 || home.status >= 500) {
+        out.push(F("site-unreachable", "critical", "read", "P1", "The homepage does not answer", `${ctx.homeUrl} returned ${home?.status ? `HTTP ${home.status}` : `no response (${home?.error ?? "error"})`}. Nothing else can be audited until the site answers.`, { urls: [ctx.homeUrl], fix: "Check DNS, the TLS certificate and the server/Worker logs; retry the audit.", accept: `curl -sI ${ctx.homeUrl} | head -1 shows HTTP 200` }));
+        return;
+    }
+    if (home.status !== 200) {
+        out.push(F("site-home-status", "high", "read", "P11", `The homepage returns HTTP ${home.status}`, `After following redirects the homepage answered ${home.status}. Crawlers treat a non-200 homepage as missing.`, { urls: [home.url], fix: "Serve the homepage with 200 (redirect chains should end on a 200 page).", accept: `curl -sI ${home.url} | head -1 shows 200` }));
+    }
+    if (ctx.startChain.length > 1) {
+        const last = ctx.startChain[ctx.startChain.length - 1];
+        const hops = ctx.startChain.filter((h) => h.location).map((h) => `${h.status} → ${h.location}`);
+        const temp = ctx.startChain.some((h) => h.location && ![301, 308].includes(h.status));
+        out.push(F("site-start-redirect", temp ? "medium" : "info", "read", "P11", temp ? "The site URL redirects with a temporary redirect" : "The site URL redirects", `${ctx.inputUrl} → ${last.url} (${hops.join(", ")}). The audit continued on ${ctx.origin}.${temp ? " Temporary redirects (302/307) keep both addresses in play; use 301/308." : ""}`, { fix: "Use the final address as the canonical site URL everywhere (config, sitemap, canonical tags).", accept: "Requesting the site URL returns 200 directly, or a single 301 to it." }));
+    }
+    // http → https
+    const h = ctx.https;
+    if (h) {
+        if (h.servesContent) {
+            out.push(F("https-no-redirect", "high", "read", "P11", "http:// serves the site instead of redirecting to https", `${h.from} answered ${h.status} with content. Two copies of every page compete, and visitors can land on the insecure one.`, { urls: [h.from], fix: "Redirect every http:// request to https:// with a 301 (Cloudflare: SSL/TLS → Edge Certificates → Always Use HTTPS; Vercel/Netlify do it automatically).", accept: `curl -sI ${h.from} shows 301 and location: https://…` }));
+        }
+        else if (h.ok === false && h.location) {
+            out.push(F("https-bad-redirect", "high", "read", "P11", "http:// does not redirect to https", `${h.from} → ${h.status} ${h.location}.`, { urls: [h.from], fix: "Redirect http:// to the https:// version of the same URL (301).", accept: `curl -sI ${h.from} shows 301 to https://` }));
+        }
+        else if (h.ok && !h.permanent) {
+            out.push(F("https-temp-redirect", "medium", "read", "P11", "http → https uses a temporary redirect", `${h.from} → ${h.status} ${h.location}. Temporary redirects (302/307) don't consolidate the two addresses.`, { urls: [h.from], fix: "Use 301 (or 308) for http → https.", accept: `curl -sI ${h.from} shows 301 or 308` }));
+        }
+    }
+    // www ↔ apex
+    const a = ctx.hostAlt;
+    if (a) {
+        if (a.servesContent) {
+            out.push(F("host-duplicate", "high", "read", "P11", `Both ${a.altHost} and ${ctx.host} serve the site`, `https://${a.altHost}/ answered ${a.status} with a page instead of redirecting. Google sees two sites and picks a canonical itself.`, { urls: [a.from], fix: `Redirect https://${a.altHost}/* to https://${ctx.host}/* with a 301 (keep the path).`, accept: `curl -sI https://${a.altHost}/ shows 301 to https://${ctx.host}/` }));
+        }
+        else if (a.ok === false && a.status >= 300 && a.status < 400) {
+            out.push(F("host-bad-redirect", "medium", "read", "P11", `${a.altHost} redirects somewhere other than ${ctx.host}`, `${a.from} → ${a.status} ${a.location ?? "?"}.`, { urls: [a.from], fix: `Redirect ${a.altHost} straight to https://${ctx.host} (one hop, 301).`, accept: `curl -sI ${a.from} shows 301 to https://${ctx.host}/` }));
+        }
+        else if (a.ok && !a.permanent) {
+            out.push(F("host-temp-redirect", "medium", "read", "P11", `${a.altHost} → ${ctx.host} uses a temporary redirect`, `${a.from} → ${a.status}.`, { urls: [a.from], fix: "Use a 301/308 so the two host names consolidate.", accept: `curl -sI ${a.from} shows 301 or 308` }));
+        }
+    }
+    if (home.status === 200 && !home.headers["strict-transport-security"]) {
+        out.push(F("hsts-missing", "low", "read", "P11", "No HSTS header", "The homepage has no Strict-Transport-Security header. The book's reference site uses HSTS for a year.", { urls: [home.url], fix: "Send `Strict-Transport-Security: max-age=31536000` (add includeSubDomains once every subdomain is https).", accept: `curl -sI ${home.url} | grep -i strict-transport-security` }));
+    }
+    if (home.status === 200 && !home.headers["content-encoding"]) {
+        const noTransform = /no-transform/i.test(home.headers["cache-control"] ?? "");
+        out.push(F("compression-missing", "medium", "read", "P14", "HTML is sent uncompressed", `The homepage answered without Content-Encoding although the request accepted br and gzip${noTransform ? "; the response carries Cache-Control: no-transform, which stops Cloudflare from compressing (the book's mistake #10)" : ""}. Pages go out several times larger than needed.`, { urls: [home.url], fix: noTransform ? "Remove `no-transform` from Cache-Control on HTML, or compress in the Worker/server yourself." : "Enable Brotli/gzip for text/html at the CDN or server; re-check after every technical fix.", accept: `curl -s -o /dev/null -D - -H 'Accept-Encoding: br, gzip' ${home.url} | grep -i content-encoding shows br or gzip` }));
+    }
+    const s = ctx.soft404;
+    if (s) {
+        if (s.soft404) {
+            out.push(F("soft-404", s.noindex ? "low" : "high", "read", "P11", s.noindex ? "Unknown URLs return 200 (with noindex)" : "Unknown URLs return 200 (soft 404)", `${s.url} answered ${s.status}${s.noindex ? " with noindex (acceptable to Google, but a real 404 is cleaner)" : ""}. Google counts a \"not found\" page served with 200 as a soft 404; AI crawlers waste fetches on it${s.noindex ? "" : " and may index junk URLs"}.`, { urls: [s.url], fix: "Return a real 404 status for unknown paths (SPA: a server/edge rule or a 404.html; Cloudflare Workers assets: not_found_handling \"404-page\" instead of \"single-page-application\"), or add noindex to the not-found screen.", accept: `curl -s -o /dev/null -w "%{http_code}" ${s.url} prints 404` }));
+        }
+        else if (s.status >= 300 && s.status < 400) {
+            out.push(F("soft-404-redirect", "medium", "read", "P11", "Unknown URLs redirect instead of returning 404", `${s.url} → ${s.status} ${s.location ?? ""}. Redirecting missing pages (often to the homepage) is treated as a soft 404.`, { urls: [s.url], fix: "Return 404 for unknown paths; keep 301s only for real moved pages.", accept: `curl -s -o /dev/null -w "%{http_code}" ${s.url} prints 404` }));
+        }
+    }
+    const l = ctx.llms;
+    if (l) {
+        if (l.isHtml) {
+            out.push(F("llms-html", "low", "quote", "P8", "/llms.txt returns an HTML page", "The address answers 200 with HTML (usually the SPA shell), so it looks like a file exists when it doesn't — the same soft-404 pattern.", { urls: [l.url], fix: "Either publish a real llms.txt (text/plain) or let the path return 404.", accept: `curl -sI ${l.url} shows 404, or content-type text/plain with the file` }));
+        }
+        else if (!l.exists) {
+            out.push(F("llms-missing", "info", "quote", "P8", "No llms.txt", "Optional. Google ignores llms.txt and most files get no requests from AI bots; it is still a cheap, consistent summary for agents that do read it.", { urls: [l.url], fix: "If you add one: `# Brand`, then `> the one sentence`, the not-X note, prices and the 10–20 most important links (book appendix B).", accept: "llms.txt and the pages do not contradict each other on any price or claim" }));
+        }
+        else {
+            if (l.bytes > 50_000) {
+                out.push(F("llms-too-big", "low", "quote", "P8", `llms.txt is ${Math.round(l.bytes / 1024)} KB`, "Keep llms.txt short (the book's example grew to 65 KB and had to be trimmed): main product, pricing, one link per language; move long lists to llms-full.txt.", { urls: [l.url], fix: "Trim llms.txt to the essentials; put the long version in llms-full.txt.", accept: "llms.txt under ~50 KB" }));
+            }
+            if (!/^\s*#\s+\S/m.test(l.text.split("\n").slice(0, 5).join("\n"))) {
+                out.push(F("llms-format", "info", "quote", "P8", "llms.txt does not start with a `# Name` heading", "The llmstxt.org format starts with an H1 (`# Brand`) followed by a `> summary` blockquote.", { urls: [l.url], fix: "Start with `# Brand` and `> the one sentence` (book appendix B).", accept: "The first line is `# <brand>` and a `>` summary follows" }));
+            }
+        }
+    }
+    for (const p of ctx.previews.filter((x) => !x.ok)) {
+        out.push(F("preview-indexable", "high", "read", "P11", `Preview copy is open and indexable: ${p.url}`, `${p.how}. Google may treat the preview as the original and pick the wrong canonical; robots.txt alone does not fix this.`, { urls: [p.url], fix: "Make the preview 301 to the main domain, send `X-Robots-Tag: noindex`, or put it behind a password (Vercel Deployment Protection, Cloudflare Access, Netlify password).", accept: `curl -sI ${p.url} shows 301 to https://${ctx.host}/ or x-robots-tag: noindex` }));
+    }
+    if (ctx.cdn === "cloudflare" && !(ctx.bots?.probes ?? []).some((b) => b.blocked)) {
+        out.push(F("cdn-cloudflare-policies", "info", "read", "P14", "Cloudflare: verify the AI bot policies (can't be seen from outside)", "Spoofed-UA tests passed, but Cloudflare verifies real crawlers by IP, so its verified-bot policies can't be tested from here. Since 15 Sep 2026, Training = \"Block\" (or \"Block on pages with ads\") also blocks Googlebot, Bingbot and Applebot.", { fix: cloudflareFix(), accept: "Security → Settings shows Search = Allow, Agent = Allow, Training ≠ Block; Search Console crawl stats show no 403s" }));
+    }
+}
+function robotsFindings(ctx, out) {
+    const r = ctx.robots;
+    if (!r)
+        return;
+    const url = r.url;
+    if (r.status >= 500) {
+        out.push(F("robots-5xx", "critical", "read", "P4", `robots.txt returns HTTP ${r.status}`, "Google treats a server error on robots.txt as \"disallow everything\" and stops crawling until it recovers.", { urls: [url], fix: "Serve robots.txt with 200 (or 404 if you have none).", accept: `curl -sI ${url} shows 200` }));
+        return;
+    }
+    if (r.isHtml) {
+        out.push(F("robots-html", "high", "read", "P4", "robots.txt returns an HTML page", "The path answers 200 with HTML (typically the SPA fallback). Crawlers find no rules and no Sitemap line; Lighthouse reports robots.txt as invalid.", { urls: [url], fix: "Publish a real robots.txt as a static file (e.g. public/robots.txt): allow the AI bots by name, close only app paths, add the Sitemap line (book appendix A).", accept: `curl -s ${url} prints User-agent lines, not HTML` }));
+        return;
+    }
+    if (!r.exists) {
+        out.push(F("robots-missing", "medium", "read", "P4", "No robots.txt", `${url} ${r.status ? `returned ${r.status}` : "did not answer (timeout or connection error)"}. Not required (404 = no restrictions), but the book's file names each AI bot, closes only app paths and points to the sitemap.`, { urls: [url], fix: "Add robots.txt (book appendix A): a group per AI bot, app paths closed, `Sitemap: https://…/sitemap.xml`.", accept: "robots.txt has no `Disallow: /` for *, names the AI bots and has a Sitemap line" }));
+        return;
+    }
+    if (r.blocksAll) {
+        out.push(F("robots-blocks-all", "critical", "read", "P4", "robots.txt blocks the whole site (Disallow: / for *)", "Every crawler without its own group is told to stay out, including Brave's crawler (which follows your * and Googlebot rules).", { urls: [url], fix: "Replace `Disallow: /` in the `User-agent: *` group with the app paths only (/app/, /admin/, /api/).", accept: "robots.txt has no `Disallow: /` for *; katman_audit_site shows no blocked tokens" }));
+    }
+    const blocked = r.blockedTokens;
+    const searchBlocked = blocked.filter((t) => {
+        const p = crawlerByToken(t)?.purpose;
+        return p === "search" || p === "user" || p === "search+training";
+    });
+    const trainingBlocked = blocked.filter((t) => !searchBlocked.includes(t));
+    if (searchBlocked.length && !r.blocksAll) {
+        out.push(F("robots-ai-search-blocked", "critical", "read", "P4", `robots.txt blocks AI search/answer bots: ${listSome(searchBlocked, 6)}`, "These bots fill the assistants' search indexes or open your page when a user asks. OpenAI: a site that blocks OAI-SearchBot is not shown in ChatGPT search answers.", { urls: [url], evidence: searchBlocked.map((t) => `${t}: Disallow: /`), fix: "Give each search/user bot its own group with `Allow: /` and only the app paths disallowed (book appendix A).", accept: "OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot and Perplexity-User may fetch /" }));
+    }
+    if (r.cloudflareManaged) {
+        out.push(F("robots-cloudflare-managed", "high", "read", "P4", "Cloudflare's managed robots.txt is on", `Cloudflare prepends its own block (Content-Signal ai-train=no and Disallow: / for ${listSome(r.managedBlocked.length ? r.managedBlocked : ["GPTBot", "ClaudeBot", "CCBot", "Google-Extended"], 8)}). Blocking Google-Extended also removes the site from Gemini grounding. The katman.pro audit found exactly this on 30 Aug 2026.`, { urls: [url], fix: `Turn off Managed robots.txt and publish your own file (book appendix A). ${cloudflareFix()}`, accept: "robots.txt no longer contains \"BEGIN Cloudflare Managed content\"; the training decision is written in your own file" }));
+    }
+    else if (trainingBlocked.length) {
+        const ge = trainingBlocked.some((t) => t.toLowerCase() === "google-extended");
+        out.push(F("robots-ai-training-blocked", ge ? "high" : "medium", "read", "P4", `robots.txt blocks training bots: ${listSome(trainingBlocked, 6)}`, `A business decision, not an error — make sure it's deliberate. OpenAI: blocking GPTBot does not affect ChatGPT search. ${ge ? "Google-Extended also controls grounding in Gemini apps: blocking it removes you from Gemini answers (not from Google Search)." : ""}`.trim(), { urls: [url], fix: "If unintended, allow them (book appendix A: same rules as the other bots). If intended, keep Google-Extended allowed and make the CDN say the same thing.", accept: "Every Disallow: / for an AI bot is written on purpose and the CDN setting matches robots.txt" }));
+    }
+    if (r.invalidLines.length) {
+        const cs = r.contentSignals.length > 0;
+        out.push(F("robots-invalid-lines", cs ? "low" : "low", "read", "P4", `robots.txt has ${r.invalidLines.length} invalid line${r.invalidLines.length === 1 ? "" : "s"}${cs ? " (Content-Signal)" : ""}`, cs
+            ? "Content-Signal: is not a rule Google or Lighthouse knows; Lighthouse reports \"robots.txt is not valid\" (the book's mistake #9). With all signals \"yes\" it means the same as no signal."
+            : "Lines crawlers don't understand are ignored; Lighthouse and Search Console report them.", { urls: [url], evidence: r.invalidLines.slice(0, 6), fix: cs ? "Write the content signals as a comment: `# Content signals: search=yes, ai-input=yes, ai-train=yes` (book appendix A)." : "Fix or remove the listed lines.", accept: "Lighthouse SEO shows \"robots.txt is valid\"" }));
+    }
+    const paths = [...new Set([...ctx.sitemap.entries.map((e) => safeUrl(e.loc)?.pathname ?? ""), ...ctx.pages.map((p) => safeUrl(p.url)?.pathname ?? "")])].filter(Boolean);
+    for (const b of r.barePrefixes) {
+        const hit = prefixCollateral(b.pattern, paths);
+        if (hit.length) {
+            out.push(F("robots-bare-prefix", "high", "read", "P4", `\`Disallow: ${b.pattern}\` also blocks ${listSome(hit, 3)}`, `A rule without a trailing slash is a prefix: \`${b.pattern}\` blocks every path that starts with it (group: ${b.agents.join(", ")}; line ${b.line}). ${hit.length} known URL${hit.length === 1 ? "" : "s"} are caught by accident.`, { urls: hit.slice(0, 10).map((p) => ctx.origin + p), fix: `Write \`Disallow: ${b.pattern}/\` (and \`Disallow: ${b.pattern}$\` if the bare path must be closed too).`, accept: `None of the site's public URLs is disallowed by \`${b.pattern}\`` }));
+        }
+        else if (b.pattern.replace(/^\//, "").length <= 2) {
+            out.push(F("robots-short-prefix", "low", "read", "P4", `\`Disallow: ${b.pattern}\` is a bare prefix`, `It blocks every future path that starts with "${b.pattern}" (e.g. ${b.pattern}ools, ${b.pattern}-guide). Line ${b.line}.`, { urls: [url], fix: `Use \`Disallow: ${b.pattern}/\` (book appendix A: "Disallow: /t hem /tr'yi hem /tools'u kapatır").`, accept: "No bare short prefixes in robots.txt" }));
+        }
+    }
+    // `/dashboard/` leaves the bare /dashboard (often a login wall) open; only app/auth paths matter here.
+    // Only where the bare path is typically a live login wall; `/t/` for a tracking path is exactly the book's advice.
+    const slash = r.slashOnly.filter((s) => ["app", "auth", "checkout"].includes(classifyPath(s.pattern.slice(0, -1))));
+    if (slash.length) {
+        out.push(F("robots-slash-only", "low", "read", "P4", "Disallow rules with a trailing slash leave the bare app path open", `${slash.map((s) => `\`${s.pattern}\``).join(", ")} don't cover ${slash.map((s) => s.pattern.slice(0, -1)).join(", ")} itself (a real case: bingbot made 242 requests to a bare /dashboard login wall).`, { urls: [url], fix: `Add \`Disallow: ${slash[0].pattern.slice(0, -1)}$\` next to \`${slash[0].pattern}\`.`, accept: "Both /x and /x/… are covered for app paths" }));
+    }
+    if (!r.sitemaps.length) {
+        out.push(F("robots-no-sitemap", "low", "read", "P4", "robots.txt has no Sitemap line", "Crawlers that read robots.txt first (Bing, AI bots) learn the sitemap address from it.", { urls: [url], fix: `Add \`Sitemap: ${ctx.origin}/sitemap.xml\` at the end of robots.txt.`, accept: "robots.txt ends with a Sitemap: line" }));
+    }
+    const conflict = ctx.pages.filter((p) => p.noindex && p.robotsBlocked.includes("Googlebot"));
+    if (conflict.length) {
+        out.push(F("robots-noindex-conflict", "medium", "read", "P4", "noindex pages are also disallowed in robots.txt", "Google can't fetch a disallowed page, so it never sees the noindex and may still list the URL (book item 01).", { urls: conflict.map((p) => p.url), fix: "Remove the Disallow for pages that carry noindex (keep noindex).", accept: "No page is both noindex and disallowed" }));
+    }
+}
+function sitemapFindings(ctx, out) {
+    const sm = ctx.sitemap;
+    const total = sm.entries.length;
+    if (!total) {
+        const tried = sm.sources.map((s) => `${s.url} → ${s.kind === "html" ? "HTML page" : s.status}`).join(", ");
+        out.push(F("sitemap-missing", "high", "read", "P4", "No sitemap found", `No sitemap with URLs was found (${tried || "nothing declared"}). The sitemap is how Google, Bing and IndexNow learn every page; the book's P4 generates it at build time.`, { fix: "Generate sitemap.xml from the page list at build time (only 200, canonical, indexable URLs; real lastmod) and reference it in robots.txt.", accept: "sitemap.xml lists every public page; every URL in it returns 200 and is canonical" }));
+        return;
+    }
+    const other = sm.entries.filter((e) => {
+        const x = safeUrl(e.loc);
+        return !x || x.hostname.toLowerCase() !== ctx.host.toLowerCase() || x.protocol !== "https:";
+    });
+    if (other.length) {
+        out.push(F("sitemap-other-host", "high", "read", "P4", `${other.length} sitemap URL${other.length === 1 ? " is" : "s are"} on another host or protocol`, `The sitemap must list the canonical https://${ctx.host} addresses; e.g. ${listSome(other.map((e) => e.loc), 3)}.`, { urls: other.slice(0, 10).map((e) => e.loc), fix: `Build sitemap URLs from the canonical origin https://${ctx.host}.`, accept: `Every <loc> starts with https://${ctx.host}/` }));
+    }
+    if (ctx.lastmod.identical || ctx.lastmod.stampedToday) {
+        out.push(F("sitemap-fake-lastmod", "medium", "read", "P4", ctx.lastmod.identical ? `Every sitemap URL has the same lastmod (${ctx.lastmod.commonDay})` : "Sitemap lastmod is today's date on almost every URL", "lastmod must change only when the page's words change. A date stamped on every build (or the launch day everywhere) tells Google and assistants the dates mean nothing — the book's mistake #1.", { evidence: [`${ctx.lastmod.withLastmod}/${total} URLs with lastmod, ${ctx.lastmod.distinctDays} distinct day(s)`], fix: "Store a per-page updated date in the page data and emit it as lastmod (and as dateModified in Article JSON-LD).", accept: "lastmod differs between pages and only moves when a page's text changes" }));
+    }
+    else if (ctx.lastmod.withLastmod === 0 && total >= 5) {
+        out.push(F("sitemap-no-lastmod", "low", "read", "P4", "The sitemap has no lastmod dates", "Google and Bing use an accurate lastmod to schedule re-crawls; IndexNow submissions also key on it.", { fix: "Emit a real lastmod per URL (the date the page's text last changed).", accept: "Every <url> has a truthful <lastmod>" }));
+    }
+    if (ctx.lastmod.future) {
+        out.push(F("sitemap-future-lastmod", "low", "read", "P4", `${ctx.lastmod.future} sitemap lastmod date(s) are in the future`, "Future dates are ignored or distrusted.", { fix: "Use the real modification date.", accept: "No lastmod after today" }));
+    }
+    const skipped = sm.sources.filter((s) => s.kind === "gzip-skipped");
+    if (skipped.length) {
+        out.push(F("sitemap-gz-skipped", "info", "read", "P4", "Gzipped sitemaps were not read", `${skipped.map((s) => s.url).join(", ")}: Katman reads plain XML sitemaps only; URL checks below cover the other sitemaps.`, {}));
+    }
+    const inSm = ctx.pages.filter((p) => p.inSitemap);
+    const notFound = inSm.filter((p) => p.status === 404 || p.status === 410);
+    const redir = inSm.filter((p) => p.status >= 300 && p.status < 400);
+    const errors = inSm.filter((p) => p.status === 0 || p.status >= 500 || (p.status >= 400 && p.status !== 404 && p.status !== 410));
+    if (notFound.length) {
+        out.push(F("sitemap-404", "high", "read", "P4", `${notFound.length} sitemap URL${notFound.length === 1 ? "" : "s"} return 404`, "Large crawler studies find ~35% of AI-crawler fetches hitting 404s; every dead URL in the sitemap spends crawl budget and signals a stale site.", { urls: notFound.map((p) => p.url), fix: "Remove dead URLs from the sitemap (or restore the pages / 301 them to the new address).", accept: "Every sitemap URL returns 200" }));
+    }
+    if (redir.length) {
+        out.push(F("sitemap-redirects", "medium", "read", "P4", `${redir.length} sitemap URL${redir.length === 1 ? " redirects" : "s redirect"}`, `The sitemap should list final URLs only; e.g. ${redir.slice(0, 3).map((p) => `${u(ctx, p.url)} → ${p.status} ${p.location ? u(ctx, p.location) : ""}`).join(", ")}.`, { urls: redir.map((p) => p.url), fix: "List the redirect targets instead (often a trailing-slash or www mismatch).", accept: "Every sitemap URL returns 200 without a redirect" }));
+    }
+    if (errors.length) {
+        out.push(F("sitemap-errors", "high", "read", "P4", `${errors.length} sitemap URL${errors.length === 1 ? "" : "s"} fail (errors/timeouts)`, `e.g. ${errors.slice(0, 3).map((p) => `${u(ctx, p.url)} → ${p.status || p.error}`).join(", ")}.`, { urls: errors.map((p) => p.url), fix: "Fix the server errors or remove those URLs.", accept: "Every sitemap URL returns 200" }));
+    }
+    const noidx = inSm.filter((p) => htmlOk(p) && p.noindex);
+    if (noidx.length) {
+        out.push(F("sitemap-noindex", "high", "read", "P4", `${noidx.length} noindex page${noidx.length === 1 ? " is" : "s are"} in the sitemap`, "The sitemap says \"index this\" while the page says \"don't\"; Search Console reports it as an error.", { urls: noidx.map((p) => p.url), fix: "Remove noindex pages (login, signup, checkout, thank-you, search) from the sitemap — or remove the noindex if the page is public.", accept: "No sitemap URL carries noindex" }));
+    }
+    const nonCanon = inSm.filter((p) => indexable(p) && p.html?.canonical && canonicalSelf(p) === false);
+    if (nonCanon.length) {
+        out.push(F("sitemap-non-canonical", "medium", "read", "P4", `${nonCanon.length} sitemap URL${nonCanon.length === 1 ? "'s" : "s'"} canonical points elsewhere`, `e.g. ${nonCanon.slice(0, 3).map((p) => `${u(ctx, p.url)} → ${u(ctx, p.html.canonical)}`).join(", ")}.`, { urls: nonCanon.map((p) => p.url), fix: "List only canonical URLs in the sitemap, or make the canonical point to the page itself.", accept: "Every sitemap URL's canonical is itself" }));
+    }
+    if (ctx.robots?.exists) {
+        const blocked = sm.entries.filter((e) => !ctx.robots.isAllowed(e.loc, "Googlebot")).map((e) => e.loc);
+        if (blocked.length) {
+            out.push(F("sitemap-robots-blocked", "high", "read", "P4", `${blocked.length} sitemap URL${blocked.length === 1 ? " is" : "s are"} disallowed in robots.txt`, "The sitemap invites crawlers to URLs that robots.txt forbids.", { urls: blocked.slice(0, 20), fix: "Remove them from the sitemap or allow them in robots.txt.", accept: "No sitemap URL is disallowed for Googlebot" }));
+        }
+    }
+}
+function botFindings(ctx, out) {
+    const t = ctx.bots;
+    if (!t)
+        return;
+    const cf = ctx.cdn === "cloudflare";
+    if (t.browserBlocked) {
+        out.push(F("bots-everyone-challenged", "critical", "read", "P14", "Every non-JavaScript visitor is refused or challenged", `Even a normal browser user agent got ${t.browserBlocked}. Crawlers can't solve JavaScript challenges, so no bot can read the site${cf ? " (Cloudflare: \"I'm Under Attack\" mode, Bot Fight Mode or a managed challenge rule)" : ""}.`, { urls: [t.url], fix: cf ? cloudflareFix() : genericCdnFix(ctx.cdn), accept: `curl -s -o /dev/null -w '%{http_code}' ${t.url} prints 200` }));
+        return;
+    }
+    const blocked = t.probes.filter((p) => p.blocked);
+    const different = t.probes.filter((p) => !p.blocked && p.different);
+    if (blocked.length) {
+        const ai = blocked.filter((p) => !p.crawler.ipVerified);
+        const engines = blocked.filter((p) => p.crawler.ipVerified);
+        const gateHit = ai.some((p) => GATE_BOTS.has(p.crawler.token));
+        const robotsAllows = (tok) => !ctx.robots?.exists || ctx.robots.isAllowed(ctx.origin + "/", tok);
+        const trainingMismatch = ai.some((p) => p.crawler.purpose === "training" && robotsAllows(p.crawler.token));
+        const severity = gateHit || trainingMismatch ? "critical" : engines.length ? "high" : "medium";
+        const parts = [];
+        if (ai.length)
+            parts.push(`${ai.map((p) => `${p.crawler.token} (${p.crawler.purpose}) → ${p.reason}`).join("; ")}.`);
+        if (trainingMismatch)
+            parts.push("robots.txt allows these training bots but the CDN/firewall refuses them — the two must say the same thing (book ch. 6).");
+        if (engines.length) {
+            parts.push(`${engines.map((p) => p.crawler.token).join(", ")} refused too${cf ? ": Cloudflare's Training policy set to \"Block\" or \"Block on pages with ads\" blocks Googlebot, Bingbot and Applebot since 15 Sep 2026" : ""}. These bots are verified by IP, so a spoofed-UA 403 can also be a fake-bot rule — confirm in Search Console → Settings → Crawl stats (403 / other 4xx) and Bing Webmaster Tools.`);
+        }
+        else if (cf && ai.length) {
+            parts.push("Googlebot, Bingbot and Applebot passed this spoofed test, but Cloudflare identifies the real ones by IP: if the block comes from the Training policy set to \"Block\" (or \"Block on pages with ads\"), Cloudflare also refuses the verified Googlebot, Bingbot and Applebot since 15 Sep 2026 — check Search Console → Settings → Crawl stats for 403s today.");
+        }
+        parts.push("Spoofed user agents only test UA-based rules; IP-verified rules can differ for the real crawlers.");
+        out.push(F("bots-blocked", severity, "read", "P14", `${blocked.length} crawler${blocked.length === 1 ? " is" : "s are"} refused at the door (${blocked.map((p) => p.crawler.token).join(", ")})`, parts.join(" "), { urls: [t.url], evidence: blocked.map((p) => `${p.crawler.token}: ${p.status} ${p.reason ?? ""}`.trim()), fix: cf ? cloudflareFix() : genericCdnFix(ctx.cdn), accept: "GPTBot, OAI-SearchBot, ClaudeBot and PerplexityBot user agents get HTTP 200 on the homepage (book command 5)" }));
+    }
+    if (different.length) {
+        out.push(F("bots-different-response", "medium", "read", "P14", `${different.length} bot${different.length === 1 ? " gets" : "s get"} a different response than browsers`, different.map((p) => `${p.crawler.token}: ${p.reason}`).join("; ") + ". The book's reference site served bots and Chrome byte-identical HTML.", { urls: [t.url], fix: "Serve bots the same HTML as browsers (no UA-based variants, no bot-only redirects).", accept: "Bots and browsers receive the same status and roughly the same HTML size" }));
+    }
+}
+function pageFindings(ctx, out) {
+    const pop = gatePopulation(ctx).filter((p) => htmlOk(p));
+    const idx = pop.filter((p) => !p.noindex);
+    const home = ctx.home;
+    const framework = ctx.scan?.framework.name ?? "";
+    // JS-off text (trap #1 / P2)
+    const csr = idx.filter((p) => csrSuspected(p));
+    const thin = idx.filter((p) => p.html.wordsJsOff < 100 && !csr.includes(p));
+    if (csr.length) {
+        const homeHit = home && csr.includes(home);
+        out.push(F("pages-csr", "critical", "read", "P2", `${csr.length} page${csr.length === 1 ? " shows" : "s show"} almost no text without JavaScript${homeHit ? " (incl. the homepage)" : ""}`, `${JS_RENDER_NOTE} ${csr.slice(0, 4).map((p) => `${u(ctx, p.url)}: ${p.html.wordsJsOff} words${p.html.rootShell ? " (empty #root)" : ""}`).join("; ")}.${ctx.lovableHint ? ` ${LOVABLE_CAVEAT}` : ""}`, { urls: csr.map((p) => p.url), fix: csrFixFor(framework, ctx.scan?.framework.router), accept: CSR_ACCEPT }));
+    }
+    if (thin.length) {
+        out.push(F("pages-thin", home && thin.includes(home) ? "medium" : "low", "quote", "P6", `${thin.length} page${thin.length === 1 ? " has" : "s have"} under 100 words without JavaScript${home && thin.includes(home) ? " (incl. the homepage)" : ""}`, `${thin.slice(0, 4).map((p) => `${u(ctx, p.url)}: ${p.html.wordsJsOff} words`).join("; ")}. Too little text to answer a question.`, { urls: thin.map((p) => p.url), fix: "Answer the page's question in the first 1–3 sentences and add the page anatomy (how it works, who it's not for, FAQ).", accept: "Each public page answers its question in its first paragraph, readable without JavaScript" }));
+    }
+    const noH1 = idx.filter((p) => !csr.includes(p) && !p.html.h1Text);
+    if (noH1.length) {
+        out.push(F("pages-no-h1", "high", "read", "P2", `${noH1.length} page${noH1.length === 1 ? " has" : "s have"} no H1 in the server HTML`, "Without JavaScript the page has no main heading; the H1 is where assistants look for what the page is about.", { urls: noH1.map((p) => p.url), fix: "Render the page's H1 (the customer's question or situation) in the server HTML.", accept: "Every public page has an H1 with text when JavaScript is off" }));
+    }
+    const noPara = idx.filter((p) => !csr.includes(p) && !p.html.firstParagraph && wordsAfterH1(p) < 25);
+    if (noPara.length) {
+        out.push(F("pages-no-first-paragraph", "medium", "quote", "P6", `${noPara.length} page${noPara.length === 1 ? " has" : "s have"} no first paragraph in the HTML`, "No <p> of 40+ characters was found; assistants quote the first 1–3 sentences of a page.", { urls: noPara.map((p) => p.url), fix: "Open each page with a paragraph that answers the question on its own (customer brings → we do → they get → fear removed).", accept: "The first 1–3 sentences answer the question when read alone" }));
+    }
+    const farAnswer = idx.filter((p) => p.html.h1Text && p.html.firstParagraph && p.html.answerNearH1 === false);
+    if (farAnswer.length) {
+        out.push(F("pages-answer-far-from-h1", "low", "quote", "P6", `${farAnswer.length} page${farAnswer.length === 1 ? "'s" : "s'"} first paragraph starts more than ~200 characters after the H1`, "Heuristic (practitioner report, not an official rule): ChatGPT's index reportedly keeps about 200 characters around the H1, so the answer should follow the H1 directly.", { urls: farAnswer.map((p) => p.url), evidence: farAnswer.slice(0, 2).map((p) => `${u(ctx, p.url)}: after H1 → "${(p.html.textAfterH1 ?? "").slice(0, 90)}…"`), fix: "Put the answer paragraph directly under the H1 (move badges, carousels and promo text below it).", accept: "H1 + the next ~200 characters contain the page's answer" }));
+    }
+    // Titles & descriptions (P3)
+    const uq = uniqueness(ctx);
+    const noTitle = idx.filter((p) => !p.html.title);
+    if (noTitle.length) {
+        out.push(F("titles-missing", "high", "quote", "P3", `${noTitle.length} page${noTitle.length === 1 ? " has" : "s have"} no <title> in the HTML`, "The title is the first thing search engines and assistants show.", { urls: noTitle.map((p) => p.url), fix: "Generate a unique title per page from one source file (situation first, then brand, ~60 characters).", accept: "Every page has a unique <title> in the server HTML" }));
+    }
+    const dupTitles = [...uq.title.values()].filter((g) => g.length > 1);
+    if (dupTitles.length) {
+        const n = dupTitles.reduce((s, g) => s + g.length, 0);
+        out.push(F("titles-duplicate", "high", "quote", "P3", `${n} pages share ${dupTitles.length} title${dupTitles.length === 1 ? "" : "s"}`, dupTitles.slice(0, 3).map((g) => `"${g[0].html.title.slice(0, 60)}" × ${g.length}`).join("; ") + ". Duplicate titles are what SPAs produce when every route uses index.html's title.", { urls: dupTitles.flat().map((p) => p.url), fix: "Give every page its own title from a single source (e.g. src/seo/pages.ts) and add a test that fails on duplicates (P3).", accept: "No two pages share a title" }));
+    }
+    const longTitles = idx.filter((p) => (p.html.title?.length ?? 0) > 65);
+    if (longTitles.length) {
+        out.push(F("titles-long", "low", "quote", "P3", `${longTitles.length} title${longTitles.length === 1 ? " is" : "s are"} longer than 65 characters`, longTitles.slice(0, 3).map((p) => `${p.html.title.length} chars: "${p.html.title.slice(0, 70)}…"`).join("; ") + ". Google rewrites long titles; keep ~60 and drop the \"| Brand\" suffix when over.", { urls: longTitles.map((p) => p.url), fix: "Shorten to ~60 characters (situation first, brand last); keep the long wording in the H1.", accept: "No title over 65 characters" }));
+    }
+    const noDesc = idx.filter((p) => !p.html.description);
+    if (noDesc.length) {
+        out.push(F("descriptions-missing", "high", "quote", "P3", `${noDesc.length} page${noDesc.length === 1 ? " has" : "s have"} no meta description`, "The description is where assistants and search snippets pick up the brand's one sentence.", { urls: noDesc.map((p) => p.url), fix: "Write a unique ~150–160 character description per page using the one-sentence pattern (research.md).", accept: "Every page has a unique meta description" }));
+    }
+    const dupDesc = [...uq.desc.values()].filter((g) => g.length > 1);
+    if (dupDesc.length) {
+        const n = dupDesc.reduce((s, g) => s + g.length, 0);
+        out.push(F("descriptions-duplicate", "medium", "quote", "P3", `${n} pages share ${dupDesc.length} meta description${dupDesc.length === 1 ? "" : "s"}`, dupDesc.slice(0, 2).map((g) => `"${g[0].html.description.slice(0, 70)}…" × ${g.length}`).join("; "), { urls: dupDesc.flat().map((p) => p.url), fix: "Write a page-specific description for each page from the single SEO source.", accept: "No two pages share a meta description" }));
+    }
+    const badLen = idx.filter((p) => p.html.description && (p.html.description.length < 70 || p.html.description.length > 160));
+    if (badLen.length) {
+        out.push(F("descriptions-length", "low", "quote", "P3", `${badLen.length} description${badLen.length === 1 ? " is" : "s are"} outside ~70–160 characters`, badLen.slice(0, 3).map((p) => `${u(ctx, p.url)}: ${p.html.description.length}`).join("; "), { urls: badLen.map((p) => p.url), fix: "Aim for 150–160 characters; cut at the last full sentence.", accept: "Descriptions are 70–160 characters" }));
+    }
+    const nosnip = idx.filter((p) => p.html.nosnippet);
+    if (nosnip.length) {
+        out.push(F("pages-nosnippet", "medium", "quote", "P3", `${nosnip.length} page${nosnip.length === 1 ? " forbids" : "s forbid"} snippets (nosnippet)`, "Google won't quote a nosnippet page in results, AI Overviews or AI Mode.", { urls: nosnip.map((p) => p.url), fix: "Remove nosnippet from pages you want cited.", accept: "No public page has nosnippet" }));
+    }
+    // Canonical
+    const noCanon = idx.filter((p) => !p.html.canonical);
+    if (noCanon.length) {
+        out.push(F("canonical-missing", "medium", "read", "P3", `${noCanon.length} page${noCanon.length === 1 ? " has" : "s have"} no canonical`, "Without a canonical, Google picks the preferred URL itself (and may pick a preview copy, www or a tracking-parameter variant).", { urls: noCanon.map((p) => p.url), fix: "Add a single, absolute, self-referencing canonical on every page.", accept: "Every page has exactly one absolute canonical pointing to itself" }));
+    }
+    const multi = idx.filter((p) => p.html.canonicalCount > 1);
+    if (multi.length) {
+        out.push(F("canonical-multiple", "medium", "read", "P3", `${multi.length} page${multi.length === 1 ? " has" : "s have"} more than one canonical tag`, "Google ignores all of them when they conflict.", { urls: multi.map((p) => p.url), fix: "Emit exactly one canonical (check layout + page both adding one).", accept: "One canonical per page" }));
+    }
+    const nonSelf = ctx.pages.filter((p) => {
+        if (!indexable(p) || !isContentKind(p.kind) || !p.html.canonical || canonicalSelf(p) !== false || p.inSitemap)
+            return false;
+        const x = safeUrl(p.url);
+        const c = safeUrl(p.html.canonical);
+        const queryVariant = !!x && !!c && x.search !== "" && c.search === "" && x.origin === c.origin && x.pathname.replace(/\/$/, "") === c.pathname.replace(/\/$/, "");
+        return !queryVariant;
+    });
+    if (nonSelf.length) {
+        out.push(F("canonical-non-self", "medium", "read", "P3", `${nonSelf.length} page${nonSelf.length === 1 ? "'s" : "s'"} canonical points to another URL`, nonSelf.slice(0, 4).map((p) => `${u(ctx, p.url)} → ${u(ctx, p.html.canonical)}${sameUrlExact(p.html.canonical.replace(/\/$/, ""), p.url.replace(/\/$/, "")) ? " (trailing slash only)" : ""}`).join("; "), { urls: nonSelf.map((p) => p.url), fix: "Point each canonical at the page's own final URL (same protocol, host and trailing-slash style).", accept: "Every public page's canonical equals its URL" }));
+    }
+    const rel = idx.filter((p) => p.html.canonicalRaw && !/^https?:\/\//i.test(p.html.canonicalRaw));
+    if (rel.length) {
+        out.push(F("canonical-relative", "low", "read", "P3", `${rel.length} canonical${rel.length === 1 ? " is" : "s are"} relative`, "Use absolute canonicals (https://domain/path).", { urls: rel.map((p) => p.url), fix: "Emit absolute canonical URLs.", accept: "Canonicals are absolute https URLs" }));
+    }
+    // og:image & social tags
+    const noOg = idx.filter((p) => !p.html.og.image);
+    if (noOg.length) {
+        out.push(F("og-image-missing", noOg.length === idx.length ? "medium" : "low", "quote", "P3", `${noOg.length} page${noOg.length === 1 ? " has" : "s have"} no og:image`, "Links shared in chats and social apps show an empty card; Google also uses og:image as a thumbnail source since March 2026 (the book's katman.pro had 0/40).", { urls: noOg.map((p) => p.url), fix: "Add og:image (1200×630, absolute URL) with og:image:alt, plus og:title/og:description/og:url and twitter:card=summary_large_image.", accept: "Every page has an og:image that opens in the browser" }));
+    }
+    const broken = [...ctx.ogImages.entries()].filter(([, v]) => !v.ok);
+    if (broken.length) {
+        out.push(F("og-image-broken", "medium", "quote", "P3", `${broken.length} og:image URL${broken.length === 1 ? " does" : "s do"} not load`, broken.map(([k, v]) => `${k} → ${v.status} ${v.contentType ?? ""}`).join("; "), { urls: broken.map(([k]) => k), fix: "Fix the image path (absolute https URL to a real image).", accept: "Every og:image URL returns 200 image/*" }));
+    }
+    const ogInc = idx.filter((p) => p.html.og.image && (!p.html.og.title || !p.html.og.description || !p.html.twitterCard));
+    if (ogInc.length) {
+        out.push(F("og-incomplete", "low", "quote", "P3", `${ogInc.length} page${ogInc.length === 1 ? " has" : "s have"} incomplete share tags`, "og:image is there but og:title, og:description or twitter:card is missing.", { urls: ogInc.map((p) => p.url), fix: "Emit og:title, og:description, og:url, og:image(+alt) and twitter:card from the same SEO source.", accept: "All share tags present on every page" }));
+    }
+    // JSON-LD (P5)
+    const noLd = idx.filter((p) => !p.html.jsonLdBlocks);
+    if (noLd.length === idx.length && idx.length) {
+        out.push(F("jsonld-missing", "medium", "quote", "P5", "No JSON-LD on any audited page", "Schema is not required for AI answers (Google), but Organization/WebSite (with the not-X note in disambiguatingDescription) is where machines read who you are.", { urls: [idx[0].url], fix: "Add Organization + WebSite site-wide, SoftwareApplication/Service on the product page, BreadcrumbList on sub-pages, Article on guides — generated from the same data as the page.", accept: "Rich Results Test shows no errors; every price and claim in schema appears on the page" }));
+    }
+    else if (noLd.length) {
+        out.push(F("jsonld-some-missing", "low", "quote", "P5", `${noLd.length} page${noLd.length === 1 ? " has" : "s have"} no JSON-LD`, `Other pages carry structured data, these don't: ${noLd.slice(0, 4).map((p) => u(ctx, p.url)).join(", ")}.`, {
+            urls: noLd.map((p) => p.url), fix: "Add at least BreadcrumbList (and the page type) to every public page.", accept: "Every public page has valid JSON-LD"
+        }));
+    }
+    const ldErr = idx.filter((p) => p.html.jsonLdErrors > 0);
+    if (ldErr.length) {
+        out.push(F("jsonld-errors", "high", "quote", "P5", `${ldErr.length} page${ldErr.length === 1 ? " has" : "s have"} JSON-LD that doesn't parse`, ldErr.slice(0, 3).map((p) => `${u(ctx, p.url)}: ${p.html.jsonLdErrorSamples[0] ?? "invalid JSON"}`).join("; "), { urls: ldErr.map((p) => p.url), fix: "Serialize JSON-LD with JSON.stringify from data (never hand-written strings); add a test that parses every block (book command 7).", accept: "Every application/ld+json block parses; Rich Results Test shows no errors" }));
+    }
+    if (home && htmlOk(home) && home.html.jsonLdBlocks && !home.html.jsonLdTypes.some((t) => /^(Organization|Corporation|LocalBusiness|WebSite)$/.test(t))) {
+        out.push(F("jsonld-no-org", "low", "quote", "P5", "The homepage has no Organization/WebSite schema", `Found: ${home.html.jsonLdTypes.join(", ")}.`, { urls: [home.url], fix: "Add Organization (name, url, logo, sameAs; alternateName + disambiguatingDescription for the not-X note) and WebSite (book appendix C).", accept: "Homepage JSON-LD includes Organization and WebSite" }));
+    }
+    const articles = idx.filter((p) => p.html.jsonLdTypes.some((t) => /Article|BlogPosting|NewsArticle/.test(t)) && !p.html.dateModified);
+    if (articles.length) {
+        out.push(F("date-modified-missing", "low", "quote", "P5", `${articles.length} article page${articles.length === 1 ? " has" : "s have"} no dateModified`, "Assistants favour fresh pages; the book's example shows a real update date and author on every page.", { urls: articles.map((p) => p.url), fix: "Add dateModified (the day the text last changed) and author to Article JSON-LD and show it on the page.", accept: "Every article has a truthful visible date and dateModified" }));
+    }
+    // lang / hreflang
+    const noLang = idx.filter((p) => !p.html.lang);
+    if (noLang.length) {
+        out.push(F("lang-missing", "medium", "read", "P3", `${noLang.length} page${noLang.length === 1 ? " has" : "s have"} no <html lang>`, "Search engines and assistants use it to match the page's language.", { urls: noLang.map((p) => p.url), fix: "Set <html lang=\"…\"> per page language.", accept: "Every page declares its language" }));
+    }
+    const withHl = idx.filter((p) => p.html.hreflang.length);
+    const noXd = withHl.filter((p) => !p.html.hreflang.some((h) => h.lang.toLowerCase() === "x-default"));
+    if (noXd.length) {
+        out.push(F("hreflang-no-xdefault", "low", "read", "P3", `${noXd.length} page${noXd.length === 1 ? " has" : "s have"} hreflang without x-default`, "x-default tells Google which version to show when no language matches (appendix E).", { urls: noXd.map((p) => p.url), fix: "Add an x-default alternate (usually the English or language-picker page).", accept: "Every hreflang set includes x-default" }));
+    }
+    const broken2 = [];
+    for (const p of withHl) {
+        for (const h of p.html.hreflang) {
+            if (h.lang.toLowerCase() === "x-default")
+                continue;
+            const target = ctx.byKey.get(keyOf(h.href));
+            if (!target || target === p)
+                continue;
+            if (!htmlOk(target)) {
+                broken2.push(`${u(ctx, p.url)} → ${h.lang} ${u(ctx, h.href)} (HTTP ${target.status})`);
+                continue;
+            }
+            const back = target.html.hreflang.some((b) => keyOf(b.href) === p.key);
+            if (!back)
+                broken2.push(`${u(ctx, p.url)} → ${h.lang} ${u(ctx, h.href)} (no link back)`);
+        }
+    }
+    if (broken2.length) {
+        out.push(F("hreflang-not-reciprocal", "medium", "read", "P3", `${broken2.length} hreflang pair${broken2.length === 1 ? " is" : "s are"} broken or one-way`, "Each language version must point to every other and to itself; a one-way or broken pair is ignored (book mistake #4: hreflang on an untranslated page).", { evidence: broken2.slice(0, 6), fix: "Generate hreflang from one table of translated pages (only pages that really exist in that language).", accept: "Every hreflang target returns 200 and links back" }));
+    }
+    // Images (P9)
+    const alt = idx.filter((p) => p.html.imagesMissingAlt > 0);
+    if (alt.length) {
+        const n = alt.reduce((s, p) => s + p.html.imagesMissingAlt, 0);
+        out.push(F("images-missing-alt", "low", "quote", "P9", `${n} image${n === 1 ? "" : "s"} without alt on ${alt.length} page${alt.length === 1 ? "" : "s"}`, "Informative images need a short alt text; decorative ones alt=\"\".", { urls: alt.map((p) => p.url), fix: "Add alt to every <img> (alt=\"\" for decoration) and width/height to prevent layout shift.", accept: "No <img> without alt" }));
+    }
+    const skips = idx.filter((p) => p.html.headingSkips > 0);
+    if (skips.length) {
+        const n = skips.reduce((s, p) => s + p.html.headingSkips, 0);
+        out.push(F("heading-skips", "low", "quote", "P3", `${n} heading level skip${n === 1 ? "" : "s"} on ${skips.length} page${skips.length === 1 ? "" : "s"}`, `e.g. ${skips.slice(0, 2).flatMap((p) => p.html.headingSkipSamples.slice(0, 1).map((s) => `${u(ctx, p.url)} ${s}`)).join("; ")}. Google doesn't mind; Bing and screen readers want h1 → h2 → h3.`, { urls: skips.map((p) => p.url), fix: "Keep the look, fix the tag level (e.g. footer headings h4 → h2).", accept: "0 heading skips" }));
+    }
+    const cfEmail = idx.filter((p) => p.html.cfEmailObfuscation);
+    if (cfEmail.length) {
+        out.push(F("cloudflare-email-obfuscation", "low", "quote", "P14", "Cloudflare hides the email address from bots", "Email Address Obfuscation replaces addresses with a script-decoded link (/cdn-cgi/l/email-protection), so crawlers and assistants can't read your contact address (book ch. 6).", { urls: cfEmail.map((p) => p.url), fix: "Cloudflare → Scrape Shield → Email Address Obfuscation: Off.", accept: "The contact email appears as plain text in the raw HTML" }));
+    }
+    const mixed = idx.filter((p) => p.html.mixedContent.length);
+    if (mixed.length) {
+        out.push(F("mixed-content", "medium", "read", "P11", `${mixed.length} https page${mixed.length === 1 ? " loads" : "s load"} http:// resources`, mixed.slice(0, 2).map((p) => `${u(ctx, p.url)}: ${p.html.mixedContent.slice(0, 2).join(", ")}`).join("; "), { urls: mixed.map((p) => p.url), fix: "Load every resource over https (or protocol-relative from the same host).", accept: "0 http:// sub-resources" }));
+    }
+    const noVp = idx.filter((p) => !p.html.viewport);
+    if (noVp.length) {
+        out.push(F("viewport-missing", "medium", "read", "P12", `${noVp.length} page${noVp.length === 1 ? " has" : "s have"} no viewport meta`, "Google indexes the mobile version.", { urls: noVp.map((p) => p.url), fix: "Add <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">.", accept: "Every page has a viewport meta; no horizontal scroll at 360 px" }));
+    }
+    if (home && htmlOk(home) && home.html.charsetOffset > 1024) {
+        out.push(F("charset-late", "low", "read", "P12", "The charset declaration comes late in the homepage HTML", `<meta charset> is at byte ${home.html.charsetOffset}; it must be within the first 1024 bytes (the book's mistake #21: JSON-LD came first).`, { urls: [home.url], fix: "Make <meta charset=\"utf-8\"> the first element in <head>.", accept: "<meta charset> within the first 1024 bytes" }));
+    }
+    // noindex on public pages outside the sitemap: check they are deliberate
+    const ni = pop.filter((p) => p.noindex && !p.inSitemap);
+    if (ni.length) {
+        out.push(F("pages-noindex", "medium", "read", "P4", `${ni.length} public page${ni.length === 1 ? " is" : "s are"} noindex`, "Fine for login/checkout/thank-you pages; a mistake anywhere else.", { urls: ni.map((p) => p.url), fix: "Remove noindex from pages that should be found; keep it only on private/utility pages.", accept: "Only login, signup, checkout, thank-you and search pages carry noindex" }));
+    }
+    // URL hygiene
+    const hyg = [];
+    for (const p of pop) {
+        const x = safeUrl(p.url);
+        if (!x)
+            continue;
+        const path = decodeURIComponentSafe(x.pathname);
+        const issues = [];
+        if (/[A-Z]/.test(x.pathname))
+            issues.push("uppercase");
+        if (/_/.test(x.pathname))
+            issues.push("underscore");
+        if (/%[0-9A-F]{2}/i.test(x.pathname) && /[^\x00-\x7F]/.test(path))
+            issues.push("non-ASCII");
+        if (x.pathname.length > 100)
+            issues.push(`${x.pathname.length} chars`);
+        if (/\/\//.test(x.pathname))
+            issues.push("double slash");
+        if (issues.length)
+            hyg.push(`${u(ctx, p.url)} (${issues.join(", ")})`);
+    }
+    const slashStyles = new Set(pop.filter((p) => (safeUrl(p.url)?.pathname ?? "/") !== "/").map((p) => (safeUrl(p.url)?.pathname ?? "").endsWith("/")));
+    if (slashStyles.size > 1)
+        hyg.push("mixed trailing-slash styles across URLs");
+    if (hyg.length) {
+        out.push(F("url-hygiene", "low", "read", "P11", `${hyg.length} URL hygiene issue${hyg.length === 1 ? "" : "s"}`, "Readable URLs: lowercase, hyphens, ASCII, short, one trailing-slash style (a live URL change needs a 301).", { evidence: hyg.slice(0, 8), fix: "Use lowercase-hyphenated paths for new pages; 301 old variants.", accept: "Public URLs are lowercase, hyphenated and consistent" }));
+    }
+}
+function decodeURIComponentSafe(s) {
+    try {
+        return decodeURIComponent(s);
+    }
+    catch {
+        return s;
+    }
+}
+function keyOf(href) {
+    return urlKey(href);
+}
+function linkFindings(ctx, out) {
+    // Broken internal links (404s are ~35% of AI-crawler fetches in large studies)
+    const broken = ctx.pages.filter((p) => p.linksIn.size && (p.status === 404 || p.status === 410 || p.status >= 500 || p.status === 0));
+    if (broken.length) {
+        out.push(F("links-broken", "high", "read", "P10", `${broken.length} internal link target${broken.length === 1 ? " is" : "s are"} broken`, `${broken.slice(0, 4).map((p) => `${u(ctx, p.url)} → ${p.status || "no response"} (linked from ${p.linkedFrom.slice(0, 2).map((x) => u(ctx, x)).join(", ")})`).join("; ")}. Large crawler studies find ~35% of AI-crawler fetches end on 404s.`, { urls: broken.map((p) => p.url), fix: "Fix or remove the links; 301 moved pages to their new address.", accept: "0 broken internal links (katman_audit_site links-broken is gone)" }));
+    }
+    const redir = ctx.pages.filter((p) => p.linksIn.size && p.status >= 300 && p.status < 400);
+    if (redir.length) {
+        out.push(F("links-redirect", "low", "read", "P10", `${redir.length} internal link${redir.length === 1 ? " points" : "s point"} to a redirect`, redir.slice(0, 4).map((p) => `${u(ctx, p.url)} → ${p.location ? u(ctx, p.location) : p.status}`).join("; "), { urls: redir.map((p) => p.url), fix: "Link straight to the final URL (no redirect hops).", accept: "Internal links point at 200 pages" }));
+    }
+    const offHost = new Map();
+    for (const p of ctx.pages)
+        for (const l of p.html?.links ?? [])
+            if (l.offHost && !offHost.has(l.href))
+                offHost.set(l.href, p.url);
+    if (offHost.size) {
+        out.push(F("links-off-host", "low", "read", "P11", `${offHost.size} internal link${offHost.size === 1 ? " uses" : "s use"} another host or http://`, [...offHost.entries()].slice(0, 4).map(([h, from]) => `${h} (on ${u(ctx, from)})`).join("; "), { fix: `Use https://${ctx.host} (or relative paths) for internal links.`, accept: "All internal links use the canonical host and https" }));
+    }
+    // Orphans and weakly linked pages
+    const pop = gatePopulation(ctx).filter((p) => indexable(p) && p !== ctx.home);
+    if (ctx.mode === "full") {
+        const partial = !ctx.coverage.complete;
+        const orphans = pop.filter((p) => p.linksIn.size === 0);
+        if (orphans.length) {
+            out.push(F("pages-orphan", partial ? "low" : "medium", "read", "P10", `${orphans.length} page${orphans.length === 1 ? " gets" : "s get"} no internal links${partial ? " (among the crawled pages)" : ""}`, `${partial ? `Only ${ctx.pages.length} pages were crawled, so a link may exist on an uncrawled page. ` : ""}Pages nobody links to stay \"Discovered – not indexed\" (the book's example had 35 such pages; footer links fixed it).`, { urls: orphans.map((p) => p.url), fix: "Link each page from the homepage, the footer or a related page with descriptive anchor text (<a href>, not onClick).", accept: "Orphan pages 0 (P10)" }));
+        }
+        const weak = partial ? [] : pop.filter((p) => p.linksIn.size > 0 && p.linksIn.size < 3);
+        if (weak.length) {
+            out.push(F("pages-few-links-in", "low", "read", "P10", `${weak.length} page${weak.length === 1 ? " gets" : "s get"} fewer than 3 internal links`, "Target: at least 3 internal links to every page (appendix E); weakly linked pages index slowly.", { urls: weak.map((p) => p.url), fix: "Add related-page links (e.g. 3 previous + 3 next neighbours) and footer links for key pages.", accept: "Every public page receives ≥3 internal links" }));
+        }
+        const notInSm = pop.filter((p) => !p.inSitemap && p.source === "link" && canonicalSelf(p) !== false);
+        if (notInSm.length && ctx.sitemap.entries.length) {
+            out.push(F("pages-not-in-sitemap", "low", "read", "P4", `${notInSm.length} indexable page${notInSm.length === 1 ? " is" : "s are"} missing from the sitemap`, "Found through links, return 200, indexable, but not listed in the sitemap.", { urls: notInSm.map((p) => p.url), fix: "Generate the sitemap from the full page list.", accept: "Every public page is in the sitemap" }));
+        }
+    }
+}
+function researchFindings(ctx, out) {
+    const r = ctx.research;
+    if (!r)
+        return;
+    const home = ctx.home;
+    const sentence = ctx.sentence;
+    if (sentence && home && htmlOk(home)) {
+        const fp = home.html.firstParagraph ?? "";
+        const c1 = containment(sentence, fp);
+        if (c1 < 0.5) {
+            out.push(F("research-home-first-paragraph", "high", "quote", "P6", "The homepage's first paragraph doesn't carry the brand's one sentence", `Overlap with research.json's sentence: ${pct(Math.round(c1 * 100), 100)}. First paragraph: "${fp.slice(0, 160) || "(none)"}". ChatGPT repeats the sentence it finds in the first line, the meta description and llms.txt.`, { urls: [home.url], evidence: [`sentence: "${sentence.slice(0, 200)}"`], fix: "Open the homepage with the one sentence (brings → we do → they get → fear removed), word for word as in research.md.", accept: "The homepage's first 1–3 sentences contain the one sentence verbatim" }));
+        }
+        const desc = home.html.description ?? "";
+        const c2 = containment(sentence, desc);
+        if (c2 < 0.5) {
+            out.push(F("research-home-description", "medium", "quote", "P3", "The homepage meta description doesn't match the one sentence", `Overlap: ${pct(Math.round(c2 * 100), 100)}. Description: "${desc.slice(0, 160) || "(none)"}".`, { urls: [home.url], fix: "Use the one-sentence pattern in the homepage meta description (~150–160 characters).", accept: "Homepage meta description = the one sentence (≥50% word overlap, ideally verbatim)" }));
+        }
+        if (ctx.llms?.exists) {
+            const c3 = containment(sentence, ctx.llms.text.slice(0, 3000));
+            if (c3 < 0.5) {
+                out.push(F("research-llms-sentence", "low", "quote", "P8", "llms.txt doesn't open with the one sentence", `Overlap in the first part of llms.txt: ${pct(Math.round(c3 * 100), 100)}.`, { urls: [ctx.llms.url], fix: "Put the one sentence in the `>` line right under `# Brand` (book appendix B), generated from the same data file.", accept: "llms.txt, the homepage and the meta description carry the same sentence" }));
+            }
+        }
+    }
+    const notX = r.brand.notX?.trim();
+    if (notX && !/\[FILL\]/i.test(notX) && ctx.llms?.exists && containment(notX, ctx.llms.text) < 0.5) {
+        out.push(F("research-notx-missing", "low", "quote", "P8", "The not-X note is missing from llms.txt", `research.json: "${notX.slice(0, 120)}".`, { urls: [ctx.llms.url], fix: "Add the not-X line to llms.txt, the About page, the FAQ and Organization.disambiguatingDescription.", accept: "The not-X note appears in llms.txt and Organization schema" }));
+    }
+    const missing = ctx.situations.filter((s) => s.url && s.status !== undefined && !(s.live && s.h1));
+    if (missing.length) {
+        out.push(F("research-situation-missing", "high", "fit", "P6", `${missing.length} situation page${missing.length === 1 ? " is" : "s are"} not live`, missing.map((s) => `"${s.sentence.slice(0, 50)}" → ${s.url}: ${s.status ? `HTTP ${s.status}` : "no response"}${s.noindex ? ", noindex" : ""}${s.live && !s.h1 ? ", no H1 without JS" : ""}`).join("; "), { urls: missing.map((s) => s.abs ?? s.url), fix: "Write each situation page with P6 (H1 = the customer's sentence, answer in the first 1–3 sentences) and add it to the sitemap and to related pages.", accept: "Each situation URL returns 200, is indexable, answers in its first 3 sentences and gets ≥1 internal link" }));
+    }
+    const unplanned = ctx.situations.filter((s) => !s.url);
+    if (unplanned.length) {
+        out.push(F("research-situation-no-url", "medium", "fit", "P6", `${unplanned.length} situation${unplanned.length === 1 ? " has" : "s have"} no page yet`, unplanned.slice(0, 5).map((s) => `"${s.sentence.slice(0, 60)}"`).join("; "), { fix: "Plan a URL per situation (katman_plan) and write the pages with P6.", accept: "Every situation in research.json has a live URL" }));
+    }
+    const unlinked = ctx.situations.filter((s) => s.live && s.linksIn === 0 && ctx.mode === "full");
+    if (unlinked.length) {
+        out.push(F("research-situation-not-linked", "medium", "fit", "P10", `${unlinked.length} live situation page${unlinked.length === 1 ? " gets" : "s get"} no internal links`, unlinked.map((s) => s.url).join(", "), { urls: unlinked.map((s) => s.abs ?? s.url), fix: "Link each situation page from the homepage/footer and from related pages.", accept: "Each situation page receives ≥1 internal link" }));
+    }
+}