katman-mcp 0.1.0

package/dist/tools/crawl/audit/index.js

@@ -0,0 +1,364 @@
+// katman_audit_site engine: crawl the live site the way a non-JavaScript crawler does, run the site-level checks,
+// then build findings, gates and the markdown report. Network goes through an injectable fetcher (tests use a fake).
+import { randomBytes } from "node:crypto";
+import { httpGet, mapLimit } from "../../../net.js";
+import { normalizeSiteUrl } from "../../../util.js";
+import { classifyPath } from "../common/routes.js";
+import { isAssetPath, originOf, safeUrl, sameSite, urlKey } from "../common/urls.js";
+import { buildFindings, canonicalSelf, computeGates } from "./analyze.js";
+import { detectCdn, runBotTest } from "./bots.js";
+import { PAGE_HEADERS, checkHostRedirect, checkHttpsRedirect, checkImages, checkPreview, checkSoft404, checkTextFile, followRedirects, wwwTwin, } from "./checks.js";
+import { analyzeHtml, headerNoindex } from "./html.js";
+import { analyzeRobots } from "./robots.js";
+import { lastmodStats, loadSitemaps } from "./sitemap.js";
+import { composeOneSentence } from "../../../util.js";
+export function researchSentence(r) {
+    if (!r)
+        return undefined;
+    const s = r.brand?.sentence?.trim();
+    if (s && !/\[FILL\]/i.test(s))
+        return s;
+    return composeOneSentence(r.brand?.oneLiner, 2);
+}
+function isHtmlResponse(res) {
+    const ct = res.headers["content-type"] ?? "";
+    if (/text\/html|application\/xhtml/i.test(ct))
+        return true;
+    return !ct && /^\s*</.test(res.body);
+}
+function makePage(url, res, source) {
+    const isHtml = isHtmlResponse(res);
+    const html = res.status === 200 && isHtml ? analyzeHtml(res.body, url) : undefined;
+    const hNo = headerNoindex(res.headers);
+    return {
+        url,
+        key: urlKey(url),
+        source,
+        kind: classifyPath(safeUrl(url)?.pathname ?? url),
+        status: res.status,
+        contentType: res.headers["content-type"],
+        location: res.location ? (safeUrl(res.location, url)?.href ?? res.location) : undefined,
+        error: res.error,
+        bytes: res.bytes,
+        ms: res.ms,
+        headers: res.headers,
+        isHtml,
+        html,
+        noindex: hNo || !!html?.noindexMeta,
+        headerNoindex: hNo,
+        inSitemap: false,
+        linksIn: new Set(),
+        linkedFrom: [],
+        robotsBlocked: [],
+        lovable: LOVABLE_RE.test(res.body.slice(0, 20_000)),
+    };
+}
+const LOVABLE_RE = /lovable\.(dev|app)|gpteng\.co|lovable-tagger|content=["']Lovable/i;
+export async function runAudit(o) {
+    const fetcher = o.fetcher ?? httpGet;
+    const now = o.now ?? new Date();
+    const maxUrls = Math.max(1, Math.min(500, Math.floor(o.maxUrls ?? 50)));
+    const verify = !!o.urls?.length;
+    const progress = o.onProgress ?? (() => undefined);
+    let origin = normalizeSiteUrl(o.siteUrl);
+    const redirectForms = new Map();
+    // 1. Homepage (follow redirects: the site URL may move to www/https or a language path)
+    progress(0, 6, "homepage");
+    const start = await followRedirects(origin + "/", fetcher, 5, { headers: PAGE_HEADERS });
+    const last = start.chain[start.chain.length - 1];
+    if (start.final.status > 0)
+        origin = originOf(last.url);
+    const host = new URL(origin).host;
+    const home = makePage(last.url, start.final, "home");
+    const ctx = {
+        mode: verify ? "verify" : "full",
+        maxUrls,
+        inputUrl: o.siteUrl,
+        origin,
+        host,
+        homeUrl: home.url,
+        homeKey: home.key,
+        startChain: start.chain,
+        home,
+        pages: [home],
+        byKey: new Map([[home.key, home]]),
+        sitemap: { sources: [], entries: [], declared: false, truncated: false },
+        sitemapByKey: new Map(),
+        lastmod: lastmodStats([], now),
+        previews: [],
+        ogImages: new Map(),
+        research: o.research,
+        sentence: researchSentence(o.research),
+        situations: [],
+        coverage: { sitemapTotal: 0, sitemapCrawled: 0, complete: true },
+        skippedByRobots: [],
+        scan: o.scan,
+        lovableHint: o.scan?.framework.builder === "lovable" || !!home.lovable,
+        notes: [],
+        now,
+    };
+    if (home.status === 0 || home.status >= 500) {
+        return finish(ctx, redirectForms);
+    }
+    ctx.cdn = detectCdn(home.headers);
+    // 2. robots.txt
+    progress(1, 6, "robots.txt");
+    const rb = await followRedirects(origin + "/robots.txt", fetcher, 3, { headers: { accept: "text/plain,*/*" } });
+    ctx.robots = analyzeRobots(rb.final.body, origin + "/robots.txt", rb.final.status, rb.final.headers["content-type"]);
+    // 3. Sitemaps
+    progress(2, 6, "sitemap");
+    ctx.sitemap = await loadSitemaps(origin, ctx.robots.exists ? ctx.robots.sitemaps : [], fetcher);
+    for (const e of ctx.sitemap.entries)
+        ctx.sitemapByKey.set(urlKey(e.loc), e);
+    ctx.lastmod = lastmodStats(ctx.sitemap.entries, now);
+    // 4. Crawl
+    const originUrl = new URL(origin);
+    const seen = new Set([home.key]);
+    const queue = [];
+    // ?sort=new, ?plan=pro…: sample one query variant per path instead of crawling every filter/state combination.
+    const variants = new Map();
+    const enqueue = (raw, source) => {
+        const x = safeUrl(raw, origin);
+        if (!x || !/^https?:$/.test(x.protocol) || !sameSite(x.hostname, host))
+            return;
+        x.hash = "";
+        if (source === "link") {
+            if (isAssetPath(x.pathname) || classifyPath(x.pathname) === "api")
+                return;
+            x.protocol = originUrl.protocol;
+            x.host = originUrl.host;
+            if (ctx.robots?.exists && !ctx.robots.isAllowed(x.href, "*")) {
+                if (ctx.skippedByRobots.length < 50 && !ctx.skippedByRobots.includes(x.href))
+                    ctx.skippedByRobots.push(x.href);
+                return;
+            }
+        }
+        const k = urlKey(x);
+        if (seen.has(k) || seen.size >= maxUrls)
+            return;
+        if (source === "link" && x.search) {
+            const base = urlKey(x.origin + x.pathname);
+            const n = variants.get(base) ?? 0;
+            if (n >= 1)
+                return;
+            variants.set(base, n + 1);
+        }
+        seen.add(k);
+        queue.push({ url: x.href, source });
+    };
+    // Following a URL to its own canonical form (slash / www / http) costs extra requests: capped per audit.
+    let followBudget = 25;
+    const fetchPage = async (url, source) => {
+        let cur = url;
+        let res = await fetcher(cur, { redirect: "manual", headers: PAGE_HEADERS });
+        for (let i = 0; i < 2 && followBudget > 0 && res.status >= 300 && res.status < 400 && res.location; i++) {
+            followBudget--;
+            const to = safeUrl(res.location, cur);
+            if (!to || urlKey(to) !== urlKey(url) || to.href === cur)
+                break;
+            redirectForms.set(cur, { url: cur, status: res.status, location: to.href });
+            cur = to.href;
+            res = await fetcher(cur, { redirect: "manual", headers: PAGE_HEADERS });
+        }
+        return makePage(cur, res, source);
+    };
+    if (verify) {
+        for (const u of o.urls)
+            enqueue(u, "input");
+    }
+    else {
+        for (const l of home.html?.links ?? [])
+            enqueue(l.href, "link");
+        for (const e of ctx.sitemap.entries)
+            enqueue(e.loc, "sitemap");
+    }
+    let fetched = 1;
+    while (queue.length) {
+        const batch = queue.splice(0, queue.length);
+        progress(3, 6, `pages ${fetched}/${Math.min(maxUrls, fetched + batch.length)}`);
+        const recs = await mapLimit(batch, 4, (q) => fetchPage(q.url, q.source));
+        fetched += recs.length;
+        for (const r of recs) {
+            if (ctx.byKey.has(r.key))
+                continue;
+            ctx.pages.push(r);
+            ctx.byKey.set(r.key, r);
+        }
+        if (!verify)
+            for (const r of recs)
+                for (const l of r.html?.links ?? [])
+                    enqueue(l.href, "link");
+    }
+    // Inbound links, sitemap membership, robots decisions
+    for (const p of ctx.pages) {
+        for (const l of p.html?.links ?? []) {
+            if (l.key === p.key)
+                continue;
+            const t = ctx.byKey.get(l.key);
+            if (!t)
+                continue;
+            t.linksIn.add(p.key);
+            if (t.linkedFrom.length < 5 && !t.linkedFrom.includes(p.url))
+                t.linkedFrom.push(p.url);
+        }
+    }
+    for (const p of ctx.pages) {
+        p.inSitemap = ctx.sitemapByKey.has(p.key);
+        if (ctx.robots?.exists)
+            p.robotsBlocked = ["Googlebot", "OAI-SearchBot"].filter((t) => !ctx.robots.isAllowed(p.url, t));
+    }
+    ctx.lovableHint ||= ctx.pages.some((p) => p.lovable);
+    const smSame = ctx.sitemap.entries.filter((e) => sameSite(safeUrl(e.loc)?.hostname ?? "", host));
+    ctx.coverage = {
+        sitemapTotal: smSame.length,
+        sitemapCrawled: ctx.pages.filter((p) => p.inSitemap).length,
+        complete: !ctx.sitemap.truncated && ctx.pages.filter((p) => p.inSitemap).length >= smSame.length,
+    };
+    // 5. Site-level checks
+    progress(4, 6, "site checks");
+    const token = o.token ?? randomBytes(4).toString("hex");
+    // At most 4 requests in flight, like the page crawl.
+    const twin = wwwTwin(host);
+    const [https, hostAlt, soft404, llms, llmsFull] = (await mapLimit([
+        () => checkHttpsRedirect(host, fetcher),
+        () => (twin ? checkHostRedirect(host, fetcher) : Promise.resolve({ from: "", status: 0, altHost: "", error: "skipped" })),
+        () => checkSoft404(origin, fetcher, token),
+        () => checkTextFile(origin + "/llms.txt", fetcher),
+        () => checkTextFile(origin + "/llms-full.txt", fetcher),
+    ], 4, (fn) => fn()));
+    Object.assign(ctx, { https, hostAlt: hostAlt.status === 0 ? undefined : hostAlt, soft404, llms, llmsFull });
+    if (!twin)
+        ctx.notes.push(`${host} is a subdomain: the www/apex check was skipped.`);
+    else if (hostAlt.status === 0)
+        ctx.notes.push(`${hostAlt.altHost} does not answer (${hostAlt.error ?? "no DNS/TLS"}): fine, but visitors who type it get an error.`);
+    // 6. Bot access
+    if (o.includeBotTest !== false) {
+        progress(5, 6, "AI bot access");
+        ctx.bots = await runBotTest(home.url, fetcher);
+        ctx.cdn = ctx.cdn ?? ctx.bots.cdn;
+    }
+    // 7. Previews, og:image, research situations
+    const previews = [...new Set([...(o.previewUrls ?? []), ...(o.research?.site?.previewUrls ?? [])].map((p) => (/^https?:\/\//i.test(p) ? p : `https://${p}`)))].filter((p) => safeUrl(p) && safeUrl(p).hostname.toLowerCase() !== host.toLowerCase());
+    ctx.previews = await mapLimit(previews.slice(0, 5), 4, (p) => checkPreview(p, host, fetcher, home.html?.title));
+    const ogUrls = ctx.pages.map((p) => p.html?.og.image).filter((x) => !!x && /^https?:\/\//i.test(x));
+    ctx.ogImages = await checkImages(ogUrls, fetcher, 5);
+    ctx.situations = await checkSituations(ctx, fetchPage);
+    return finish(ctx, redirectForms);
+}
+async function checkSituations(ctx, fetchPage) {
+    const list = ctx.research?.situations ?? [];
+    const out = list.map((s) => ({ id: s.id, sentence: s.sentence, url: s.url?.trim() || undefined }));
+    const todo = out.filter((s) => s.url && !/\[FILL\]/i.test(s.url)).slice(0, 10);
+    await mapLimit(todo, 4, async (s) => {
+        const abs = safeUrl(s.url, ctx.origin);
+        if (!abs)
+            return;
+        s.abs = abs.href;
+        const known = ctx.byKey.get(urlKey(abs));
+        const rec = known ?? (await fetchPage(abs.href, "situation"));
+        s.status = rec.status;
+        s.noindex = rec.noindex;
+        s.live = rec.status === 200 && rec.isHtml && !rec.noindex;
+        s.h1 = !!rec.html?.h1Text;
+        s.linksIn = known ? known.linksIn.size : 0;
+    });
+    return out;
+}
+function finish(ctx, redirectForms) {
+    // Redirecting forms of sitemap URLs and links count as redirects in the findings.
+    const findings = buildFindings(ctx);
+    addRedirectFormFindings(ctx, redirectForms, findings);
+    const gates = computeGates(ctx);
+    return { result: toAuditResult(ctx, findings, gates), ctx, findings, redirectForms };
+}
+function addRedirectFormFindings(ctx, forms, findings) {
+    if (!forms.size)
+        return;
+    const smRedirects = ctx.sitemap.entries.filter((e) => forms.has(e.loc)).map((e) => `${e.loc} → ${forms.get(e.loc).status} ${forms.get(e.loc).location}`);
+    if (smRedirects.length) {
+        findings.push({
+            id: "sitemap-redirect-forms",
+            title: `${smRedirects.length} sitemap URL${smRedirects.length === 1 ? " is" : "s are"} listed in a redirecting form (slash/www/http)`,
+            severity: "medium",
+            gate: "read",
+            prompt: "P4",
+            detail: "The sitemap should list the final canonical URL; these redirect to the same page with a different trailing slash, host or protocol.",
+            evidence: smRedirects.slice(0, 5),
+            urls: smRedirects.map((s) => s.split(" → ")[0]),
+            fix: "Emit sitemap URLs in exactly the canonical form (same trailing-slash style as the canonical tag).",
+            accept: "Every sitemap URL returns 200 directly",
+        });
+    }
+    const linked = [];
+    for (const p of ctx.pages)
+        for (const l of p.html?.links ?? [])
+            if (forms.has(l.href) && linked.length < 50)
+                linked.push(`${l.href} (on ${p.url})`);
+    if (linked.length) {
+        findings.push({
+            id: "links-redirect-forms",
+            title: `${linked.length} internal link${linked.length === 1 ? " goes" : "s go"} through a redirect (slash/www/http)`,
+            severity: "low",
+            gate: "read",
+            prompt: "P10",
+            detail: "Each link costs crawlers an extra hop.",
+            evidence: linked.slice(0, 5),
+            fix: "Link to the canonical form directly (match the trailing-slash style of the canonical tag).",
+            accept: "Internal links point at 200 URLs",
+        });
+    }
+}
+export function toAuditResult(ctx, findings, gates) {
+    const home = ctx.home;
+    return {
+        version: 1,
+        ts: ctx.now.toISOString(),
+        siteUrl: ctx.origin,
+        pages: ctx.pages.map((p) => ({
+            url: p.url,
+            status: p.status,
+            contentType: p.contentType,
+            noindex: p.noindex,
+            title: p.html?.title,
+            description: p.html?.description,
+            canonical: p.html?.canonical,
+            canonicalSelf: canonicalSelf(p),
+            h1Count: p.html?.h1Count ?? 0,
+            headingSkips: p.html?.headingSkips ?? 0,
+            wordsJsOff: p.html?.wordsJsOff ?? 0,
+            firstParagraph: p.html?.firstParagraph,
+            jsonLdTypes: p.html?.jsonLdTypes ?? [],
+            jsonLdErrors: p.html?.jsonLdErrors ?? 0,
+            ogImage: p.html?.og.image,
+            lang: p.html?.lang,
+            hreflang: p.html?.hreflang ?? [],
+            imagesMissingAlt: p.html?.imagesMissingAlt ?? 0,
+            internalLinksOut: (p.html?.links ?? []).filter((l) => l.key !== p.key).length,
+            internalLinksIn: p.linksIn.size,
+            inSitemap: p.inSitemap,
+            dateModified: p.html?.dateModified,
+            bytes: p.bytes,
+            ms: p.ms,
+        })),
+        site: {
+            httpsRedirect: ctx.https ? (ctx.https.status === 0 ? undefined : !!ctx.https.ok) : undefined,
+            hostRedirect: ctx.hostAlt ? !!ctx.hostAlt.ok : undefined,
+            hsts: home && home.status === 200 ? !!home.headers["strict-transport-security"] : undefined,
+            compression: home && home.status === 200 ? (home.headers["content-encoding"] ?? null) : undefined,
+            soft404: ctx.soft404?.soft404,
+            llmsTxt: { exists: !!ctx.llms?.exists, bytes: ctx.llms?.exists ? ctx.llms.bytes : undefined },
+            robots: {
+                exists: !!ctx.robots?.exists,
+                sitemaps: ctx.robots?.sitemaps ?? [],
+                blocksAll: !!ctx.robots?.blocksAll,
+                blockedAiTokens: ctx.robots?.blockedTokens ?? [],
+                invalidLines: ctx.robots?.invalidLines ?? [],
+            },
+            sitemapUrls: ctx.sitemap.entries.length,
+            cdn: ctx.cdn,
+            bots: (ctx.bots?.probes ?? []).map((p) => ({ token: p.crawler.token, purpose: p.crawler.purpose, status: p.status, blocked: p.blocked, reason: p.reason })),
+        },
+        gates,
+        findings,
+    };
+}

package/dist/tools/crawl/audit/model.js

@@ -0,0 +1 @@
+export {};

package/dist/tools/crawl/audit/report.js

@@ -0,0 +1,199 @@
+import { mdTable, truncate } from "../../../util.js";
+import { LOVABLE_CAVEAT } from "../common/advice.js";
+import { findingsList, rankByImpact, severityLine } from "../common/findings.js";
+import { shortUrl } from "../common/urls.js";
+import { canonicalSelf, descUnique, gatePopulation, htmlOk, titleUnique, uniqueness } from "./analyze.js";
+const Y = "✓";
+const N = "✗";
+export function topIssues(ctx, findings, n = 10) {
+    const actionable = findings.filter((f) => f.severity !== "info");
+    return rankByImpact(actionable, { pages: gatePopulation(ctx).length, homeUrl: ctx.home?.url }).slice(0, n);
+}
+export function auditMarkdown(ctx, result, findings, opts = {}) {
+    const L = [];
+    const g = result.gates;
+    const crawled = ctx.pages.length;
+    L.push(`# Katman site audit — ${ctx.origin}`);
+    L.push(`${result.ts.slice(0, 16).replace("T", " ")} UTC · ${ctx.mode === "verify" ? "quick verify" : "full audit"} · ${crawled} URL${crawled === 1 ? "" : "s"} fetched · sitemap: ${ctx.sitemap.entries.length} URL${ctx.sitemap.entries.length === 1 ? "" : "s"}${ctx.cdn ? ` · CDN: ${ctx.cdn}` : ""} · ${severityLine(findings)}`);
+    if (opts.mdPath || opts.jsonPath)
+        L.push(`Saved: ${[opts.mdPath, opts.jsonPath].filter(Boolean).join(" · ")}`);
+    L.push("");
+    // Gates
+    L.push("## Gates");
+    L.push(mdTable(["Gate", "Pass", "Main reasons"], ["read", "quote", "fit"].map((k) => [
+        k === "read" ? "Read (Okunmak)" : k === "quote" ? "Quote (Alıntılanmak)" : "Fit (Soruya uymak)",
+        g[k].total ? `${g[k].pass}/${g[k].total}` : "–",
+        g[k].notes.slice(1, 4).join("; ") || (g[k].total ? "all pass" : ""),
+    ])));
+    L.push("");
+    L.push(`- ${g.read.notes[0]}`);
+    L.push(`- ${g.quote.notes[0]}`);
+    L.push(`- ${g.fit.notes[0]}`);
+    L.push("");
+    // Top issues
+    const top = topIssues(ctx, findings);
+    L.push("## Top issues by traffic impact");
+    L.push(top.length ? findingsList(top, { urlLimit: 5 }) : "No actionable issues found.");
+    L.push("");
+    // Bots
+    if (ctx.bots) {
+        L.push("## AI bot access (homepage, user-agent test)");
+        const rows = [["Browser (baseline)", "—", "—", ctx.bots.browser.status, ctx.bots.browserBlocked ? `${N} ${ctx.bots.browserBlocked}` : `${Y} ${kb(ctx.bots.browser.bytes)}`]];
+        for (const p of ctx.bots.probes) {
+            rows.push([
+                p.crawler.token,
+                p.crawler.org,
+                p.crawler.purpose,
+                p.status || "—",
+                p.blocked ? `${N} blocked: ${p.reason ?? ""}` : p.different ? `! ${p.reason}` : `${Y} ${kb(p.bytes)}`,
+            ]);
+        }
+        L.push(mdTable(["Bot", "Org", "Purpose", "Status", "Result"], rows));
+        L.push("");
+        L.push("Spoofed user agents only test UA-based rules; CDNs verify real Googlebot/Bingbot/Applebot (and increasingly AI bots) by IP, so their real result can differ. Confirm Google in Search Console → Settings → Crawl stats.");
+        if (ctx.lovableHint)
+            L.push(`Lovable detected: ${LOVABLE_CAVEAT}`);
+        L.push("");
+    }
+    // The nine commands (book ch. 16)
+    L.push("## The book's nine checks (ch. 16), run for you");
+    L.push(mdTable(["#", "Check", "Result", "Command to re-run"], nineCommands(ctx)));
+    L.push("");
+    // P14 table
+    L.push(`## Pre-publish table (P14)${gatePopulation(ctx).length > 25 ? " — first 25 pages" : ""}`);
+    L.push(p14Table(ctx));
+    L.push("");
+    L.push("Columns: status 200 · no unwanted noindex · unique title · unique description · canonical to itself · og:image (✓ loads when checked) · JSON-LD parses (– none) · H1 + first paragraph without JavaScript · in sitemap · internal links in (target ≥ 3).");
+    L.push("");
+    // Other findings
+    const rest = findings.filter((f) => !top.includes(f));
+    if (rest.length) {
+        L.push(`## Other findings (${rest.length})`);
+        if (opts.full) {
+            L.push(findingsList(rankByImpact(rest, { pages: gatePopulation(ctx).length, homeUrl: ctx.home?.url }), { urlLimit: 10 }));
+        }
+        else {
+            for (const f of rankByImpact(rest, { pages: gatePopulation(ctx).length, homeUrl: ctx.home?.url }).slice(0, 40)) {
+                L.push(`- [${f.severity.toUpperCase()}] ${f.title} (${f.gate}${f.prompt ? ` · ${f.prompt}` : ""})${f.urls?.length ? ` — ${f.urls.length} URL${f.urls.length === 1 ? "" : "s"}` : ""}`);
+            }
+            if (rest.length > 40)
+                L.push(`- …${rest.length - 40} more in audit.md`);
+            L.push("Full details (fix + acceptance criterion for every finding): .katman/audit.md");
+        }
+        L.push("");
+    }
+    // Coverage
+    L.push("## Coverage");
+    const src = count(ctx.pages.map((p) => p.source));
+    L.push(`- Fetched ${crawled} URLs (${Object.entries(src).map(([k, v]) => `${v} ${k}`).join(", ")}); cap ${ctx.maxUrls}.`);
+    if (ctx.mode === "full") {
+        L.push(`- Sitemap coverage: ${ctx.coverage.sitemapCrawled}/${ctx.coverage.sitemapTotal} sitemap URLs fetched${ctx.coverage.complete ? "" : " — inbound-link counts are lower bounds; raise max_urls for a full check"}.`);
+    }
+    for (const pv of ctx.previews)
+        L.push(`- Preview copy ${pv.url}: ${pv.how} ${pv.ok ? Y : N}`);
+    if (ctx.skippedByRobots.length)
+        L.push(`- Not fetched (disallowed in robots.txt): ${ctx.skippedByRobots.slice(0, 5).join(", ")}${ctx.skippedByRobots.length > 5 ? " …" : ""}`);
+    if (ctx.research)
+        L.push(`- Research: ${ctx.sentence ? "one sentence checked on the homepage" : "no one sentence yet"}; ${ctx.situations.length} situation(s).`);
+    else
+        L.push("- No .katman/research.json: the fit gate and one-sentence checks were skipped (run katman_research).");
+    for (const n of ctx.notes)
+        L.push(`- ${n}`);
+    L.push("");
+    L.push("Next: fix the top issue with the prompt shown (katman_prompt), then re-verify just those URLs with katman_audit_site { urls: [...] }.");
+    return L.join("\n");
+}
+function count(xs) {
+    const o = {};
+    for (const x of xs)
+        o[x] = (o[x] ?? 0) + 1;
+    return o;
+}
+function kb(n) {
+    return n >= 1024 ? `${(n / 1024).toFixed(1)} KB` : `${n} B`;
+}
+function mark(v) {
+    return v === undefined ? "–" : v ? Y : N;
+}
+export function p14Rows(ctx) {
+    const u = uniqueness(ctx);
+    const pop = gatePopulation(ctx);
+    const ordered = [...pop].sort((a, b) => (a === ctx.home ? -1 : b === ctx.home ? 1 : 0));
+    return ordered.slice(0, 25).map((p) => {
+        const ok = htmlOk(p);
+        const h = p.html;
+        const og = h?.og.image ? (ctx.ogImages.get(h.og.image)?.ok ?? true) : false;
+        const ld = !h || !h.jsonLdBlocks ? "–" : h.jsonLdErrors ? N : Y;
+        const jsOff = !!h && !!h.h1Text && !!h.firstParagraph && h.wordsJsOff >= 100;
+        return {
+            page: p,
+            cells: [
+                truncate(shortUrl(p.url, ctx.host), 48),
+                String(p.status || "—"),
+                ok ? mark(!p.noindex) : "–",
+                ok ? mark(titleUnique(u, p)) : "–",
+                ok ? mark(descUnique(u, p)) : "–",
+                ok ? mark(canonicalSelf(p) ?? false) : "–",
+                ok ? mark(og) : "–",
+                ok ? ld : "–",
+                ok ? mark(jsOff) : "–",
+                mark(p.inSitemap),
+                ctx.mode === "verify" && p.linksIn.size === 0 ? "0*" : String(p.linksIn.size),
+            ],
+        };
+    });
+}
+function p14Table(ctx) {
+    const rows = p14Rows(ctx);
+    if (!rows.length)
+        return "No public pages were fetched.";
+    const t = mdTable(["URL", "Status", "No noindex", "Title unique", "Desc unique", "Canonical self", "og:image", "JSON-LD", "JS-off H1+¶", "In sitemap", "Links in"], rows.map((r) => r.cells));
+    return ctx.mode === "verify" ? `${t}\n\n\\* quick verify mode: only links among the checked pages are counted.` : t;
+}
+function nineCommands(ctx) {
+    const o = ctx.origin;
+    const home = ctx.home;
+    const h = home?.html;
+    const rows = [];
+    const https = ctx.https;
+    const httpsTxt = !https
+        ? "not tested"
+        : https.status === 0
+            ? `http:// not reachable (${https.error ?? "error"})`
+            : https.location
+                ? `http:// → ${https.status} ${https.location} ${https.ok && https.permanent ? Y : N}`
+                : `http:// → ${https.status} ${N}`;
+    rows.push(["1", "Homepage 200 · http → https", `${home?.status ?? "—"} ${home?.status === 200 ? Y : N} · ${httpsTxt}`, `curl -sI ${o}/ | head -1; curl -sI http://${ctx.host}/ | grep -iE "^HTTP|^location"`]);
+    const s = ctx.soft404;
+    rows.push(["2", "Unknown URL is a real 404", s ? `${s.status} ${s.status === 404 || s.status === 410 ? Y : N}${s.soft404 && s.noindex ? " (noindex)" : ""}` : "not tested", `curl -s -o /dev/null -w "%{http_code}" ${o}/does-not-exist`]);
+    const canon = home ? canonicalSelf(home) : undefined;
+    rows.push([
+        "3",
+        "Title, description, canonical, robots, og:image",
+        h
+            ? `title ${h.title ? `${h.title.length} chars ${Y}` : N} · description ${h.description ? `${h.description.length} chars ${Y}` : N} · canonical ${canon === undefined ? N : canon ? `self ${Y}` : `→ ${h.canonical} ${N}`} · robots ${h.metaRobots ?? "none"} ${home.noindex ? N : Y} · og:image ${h.og.image ? Y : N}`
+            : "—",
+        `curl -s ${o}/ | grep -oiE '<title>[^<]*|name="description" content="[^"]*|rel="canonical" href="[^"]*|name="robots" content="[^"]*|og:image" content="[^"]*'`,
+    ]);
+    rows.push(["4", "Words without JavaScript (homepage)", h ? `${h.wordsJsOff} visible words ${h.wordsJsOff >= 100 ? Y : N}` : "—", `curl -s ${o}/ | sed 's/<[^>]*>/ /g' | wc -w   (counts script text too)`]);
+    const want = ["GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot"];
+    const bots = ctx.bots?.probes.filter((p) => want.includes(p.crawler.token)) ?? [];
+    rows.push([
+        "5",
+        "AI bots get in (all 200)",
+        bots.length ? bots.map((b) => `${b.crawler.token} ${b.status}${b.blocked ? ` ${N}` : ""}`).join(" · ") : "not tested",
+        `for b in GPTBot/1.4 OAI-SearchBot/1.4 ClaudeBot/1.0 PerplexityBot/1.0; do echo "$b $(curl -s -o /dev/null -w '%{http_code}' -A "Mozilla/5.0 (compatible; $b)" ${o}/)"; done`,
+    ]);
+    const r = ctx.robots;
+    rows.push([
+        "6",
+        "robots.txt and sitemap",
+        `${r ? (r.exists ? `robots.txt 200, ${r.groups.length} groups, Sitemap line ${r.sitemaps.length ? Y : N}` : r.isHtml ? `robots.txt is HTML ${N}` : `robots.txt ${r.status} ${N}`) : "—"} · sitemap ${ctx.sitemap.entries.length} <loc> ${ctx.sitemap.entries.length ? Y : N}`,
+        `curl -s ${o}/robots.txt; curl -s ${o}/sitemap.xml | grep -c "<loc>"`,
+    ]);
+    rows.push(["7", "JSON-LD parses (homepage)", h ? (h.jsonLdBlocks ? `${h.jsonLdBlocks} block(s), ${h.jsonLdErrors ? `${h.jsonLdErrors} invalid ${N}` : `all valid ${Y}`}` : `none ${N}`) : "—", `curl -s ${o}/ | python3 -c "import sys,re,json; b=re.findall(r'application/ld\\+json\\">(.*?)</script', sys.stdin.read(), re.S); [json.loads(x) for x in b]; print(len(b), 'valid')"`]);
+    rows.push(["8", "Mobile speed + SEO (Lighthouse)", "not run by Katman (needs Chrome): targets LCP ≤ 2.5 s, CLS ≤ 0.1", `npx lighthouse ${o}/ --form-factor=mobile --only-categories=performance,seo --view`]);
+    const enc = home?.headers["content-encoding"];
+    rows.push(["9", "HTML compressed", home ? (enc ? `${enc} ${Y}` : `none ${N}`) : "—", `curl -s -o /dev/null -D - -H 'Accept-Encoding: br, gzip' ${o}/ | grep -i content-encoding`]);
+    return rows;
+}