katman-mcp 0.1.0

package/dist/tools/crawl/audit/robots.js

@@ -0,0 +1,199 @@
+// robots.txt analysis (pure). Allow/deny decisions use robots-parser (Google's matching algorithm); the line-level
+// validation mirrors Lighthouse's robots-txt audit so "Content-Signal:" and friends are reported the way site owners
+// will see them in Lighthouse / Search Console.
+import robotsParserModule from "robots-parser";
+import { CRAWLERS, PERMISSION_TOKENS, ROBOTS_TOKENS } from "../../../crawlers.js";
+const robotsParser = robotsParserModule.default ?? robotsParserModule;
+const KNOWN = new Set([
+    "user-agent", "disallow", "allow", "sitemap", "crawl-delay", "clean-param", "host", "request-rate", "visit-time", "noindex",
+]);
+/** Tokens Cloudflare's managed robots.txt blocks (developers.cloudflare.com/bots/additional-configurations/managed-robots-txt). */
+export const CF_MANAGED_TOKENS = [
+    "Amazonbot", "Applebot-Extended", "Bytespider", "CCBot", "ClaudeBot", "Google-Extended", "GPTBot", "meta-externalagent",
+];
+export function allRobotsTokens() {
+    const seen = new Map();
+    for (const t of [...ROBOTS_TOKENS, ...CRAWLERS.map((c) => c.token), ...PERMISSION_TOKENS]) {
+        if (!seen.has(t.toLowerCase()))
+            seen.set(t.toLowerCase(), t);
+    }
+    return [...seen.values()];
+}
+export function looksLikeHtml(body, contentType) {
+    return /text\/html/i.test(contentType ?? "") || /^\s*(<!doctype html|<html[\s>]|<head[\s>])/i.test(body);
+}
+export function analyzeRobots(text, robotsUrl, status = 200, contentType) {
+    let origin = "https://example.invalid";
+    try {
+        origin = new URL(robotsUrl).origin;
+    }
+    catch {
+        /* keep */
+    }
+    const isHtml = status === 200 && looksLikeHtml(text, contentType);
+    const usable = status === 200 && !isHtml;
+    const body = usable ? text : "";
+    const lines = body.split(/\r\n|\r|\n/);
+    const groups = [];
+    const invalidLines = [];
+    const contentSignals = [];
+    const sitemaps = [];
+    let current;
+    let lastWasAgent = false;
+    lines.forEach((raw, i) => {
+        const lineNo = i + 1;
+        const clean = raw.replace(/#.*$/, "").trim();
+        if (!clean)
+            return;
+        const idx = clean.indexOf(":");
+        if (idx < 0) {
+            invalidLines.push(`line ${lineNo}: "${clip(clean)}" — syntax not understood`);
+            return;
+        }
+        const key = clean.slice(0, idx).trim().toLowerCase();
+        const val = clean.slice(idx + 1).trim();
+        if (!KNOWN.has(key)) {
+            if (key === "content-signal") {
+                contentSignals.push(val);
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — unknown directive (Lighthouse: "robots.txt is not valid"; write it as a # comment)`);
+            }
+            else {
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — unknown directive`);
+            }
+            lastWasAgent = false;
+            return;
+        }
+        if (key === "user-agent") {
+            if (!val)
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — no user-agent specified`);
+            if (!lastWasAgent || !current) {
+                current = { agents: [], rules: [], line: lineNo };
+                groups.push(current);
+            }
+            if (val)
+                current.agents.push(val);
+            lastWasAgent = true;
+            return;
+        }
+        lastWasAgent = false;
+        if (key === "allow" || key === "disallow") {
+            if (!current) {
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — no user-agent specified before this rule`);
+                return;
+            }
+            if (val && !/^[/*]/.test(val)) {
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — pattern should be empty or start with "/" or "*"`);
+            }
+            const dollar = val.indexOf("$");
+            if (dollar >= 0 && dollar !== val.length - 1) {
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — "$" should only be used at the end of the pattern`);
+            }
+            if (val)
+                current.rules.push({ type: key, pattern: val, line: lineNo });
+            return;
+        }
+        if (key === "sitemap") {
+            let okUrl = false;
+            try {
+                okUrl = /^https?:$/.test(new URL(val).protocol);
+            }
+            catch {
+                okUrl = false;
+            }
+            if (!okUrl)
+                invalidLines.push(`line ${lineNo}: "${clip(clean)}" — invalid sitemap URL (must be absolute)`);
+            else
+                sitemaps.push(val);
+        }
+    });
+    const robot = robotsParser(origin + "/robots.txt", body);
+    const isAllowed = (url, token) => {
+        if (!usable)
+            return true;
+        let u;
+        try {
+            u = new URL(url, origin);
+        }
+        catch {
+            return true;
+        }
+        const o = new URL(origin);
+        u.protocol = o.protocol;
+        u.host = o.host;
+        return robot.isAllowed(u.href, token) !== false;
+    };
+    const blocksAll = usable && !isAllowed(origin + "/", "*");
+    const blockedTokens = usable ? allRobotsTokens().filter((t) => !isAllowed(origin + "/", t)) : [];
+    const managedMarker = /BEGIN Cloudflare Managed content/i.test(body);
+    const trainNo = contentSignals.some((v) => /ai-train\s*=\s*no/i.test(v));
+    const managedHits = CF_MANAGED_TOKENS.filter((t) => blockedTokens.some((b) => b.toLowerCase() === t.toLowerCase()));
+    const cloudflareManaged = managedMarker || (trainNo && managedHits.length >= 3);
+    const barePrefixes = [];
+    const slashOnly = [];
+    for (const g of groups) {
+        const dis = g.rules.filter((r) => r.type === "disallow").map((r) => r.pattern);
+        for (const r of g.rules) {
+            if (r.type !== "disallow")
+                continue;
+            const p = r.pattern;
+            if (p === "/" || /[*$?]/.test(p))
+                continue;
+            if (p.endsWith("/")) {
+                const bare = p.slice(0, -1);
+                if (bare && !dis.includes(bare) && !dis.includes(bare + "$"))
+                    slashOnly.push({ pattern: p, line: r.line, agents: g.agents });
+                continue;
+            }
+            const last = p.split("/").pop() ?? "";
+            if (/\.[a-z0-9]{2,5}$/i.test(last))
+                continue; // a file like /private.html
+            barePrefixes.push({ pattern: p, line: r.line, agents: g.agents });
+        }
+    }
+    return {
+        url: robotsUrl,
+        status,
+        exists: usable && body.trim().length > 0,
+        isHtml,
+        bytes: text.length,
+        sitemaps,
+        groups,
+        blocksAll,
+        blockedTokens,
+        invalidLines,
+        contentSignals,
+        cloudflareManaged,
+        managedBlocked: cloudflareManaged ? managedHits : [],
+        barePrefixes: dedupePrefix(barePrefixes),
+        slashOnly: dedupePrefix(slashOnly),
+        isAllowed,
+    };
+}
+function dedupePrefix(list) {
+    const seen = new Map();
+    for (const p of list) {
+        const k = p.pattern;
+        const prev = seen.get(k);
+        if (prev)
+            prev.agents = [...new Set([...prev.agents, ...p.agents])];
+        else
+            seen.set(k, { ...p, agents: [...p.agents] });
+    }
+    return [...seen.values()];
+}
+/** Paths a bare prefix blocks beyond its own subtree: `/t` → `/tools`, `/tr`. */
+export function prefixCollateral(pattern, paths) {
+    const out = new Set();
+    for (const p of paths) {
+        if (!p.startsWith(pattern))
+            continue;
+        const rest = p.slice(pattern.length);
+        if (rest === "" || rest.startsWith("/") || rest.startsWith("?") || rest.startsWith(".") || rest.startsWith("#"))
+            continue;
+        out.add(p);
+    }
+    return [...out];
+}
+function clip(s, n = 90) {
+    return s.length > n ? s.slice(0, n - 1) + "…" : s;
+}

package/dist/tools/crawl/audit/sitemap.js

@@ -0,0 +1,147 @@
+import { decodeEntities } from "../common/text.js";
+import { looksLikeHtml } from "./robots.js";
+function tag(block, name) {
+    const m = block.match(new RegExp(`<(?:[a-z0-9]+:)?${name}\\b[^>]*>([\\s\\S]*?)</(?:[a-z0-9]+:)?${name}>`, "i"));
+    return m ? decodeEntities(m[1].trim()) : undefined;
+}
+export function parseSitemapXml(xml) {
+    const body = xml.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1").replace(/<!--[\s\S]*?-->/g, "");
+    if (/<(?:[a-z0-9]+:)?sitemapindex[\s>]/i.test(body)) {
+        const children = [...body.matchAll(/<(?:[a-z0-9]+:)?sitemap\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9]+:)?sitemap>/gi)]
+            .map((m) => ({ loc: tag(m[1], "loc") ?? "", lastmod: tag(m[1], "lastmod") }))
+            .filter((c) => c.loc);
+        return { kind: "index", entries: [], children };
+    }
+    if (/<(?:[a-z0-9]+:)?urlset[\s>]/i.test(body)) {
+        const entries = [];
+        for (const m of body.matchAll(/<(?:[a-z0-9]+:)?url\b[^>]*>([\s\S]*?)<\/(?:[a-z0-9]+:)?url>/gi)) {
+            const loc = tag(m[1], "loc");
+            if (!loc)
+                continue;
+            const alternates = [];
+            for (const l of m[1].matchAll(/<xhtml:link\b([^>]*?)\/?>/gi)) {
+                const attrs = l[1];
+                const lang = attrs.match(/hreflang\s*=\s*["']([^"']+)["']/i)?.[1];
+                const href = attrs.match(/href\s*=\s*["']([^"']+)["']/i)?.[1];
+                if (lang && href)
+                    alternates.push({ lang, href: decodeEntities(href) });
+            }
+            entries.push({ loc, lastmod: tag(m[1], "lastmod"), alternates });
+        }
+        return { kind: "urlset", entries, children: [] };
+    }
+    const lines = body.split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
+    if (lines.length && lines.every((l) => /^https?:\/\/\S+$/i.test(l))) {
+        return { kind: "text", entries: lines.map((loc) => ({ loc, alternates: [] })), children: [] };
+    }
+    return { kind: "invalid", entries: [], children: [] };
+}
+/**
+ * Load sitemaps: the robots.txt Sitemap: lines, else /sitemap.xml, /sitemap_index.xml, /sitemap-index.xml (first
+ * that works). Follows index files breadth-first, at most `maxFiles` fetches.
+ */
+export async function loadSitemaps(origin, declared, fetcher, opts = {}) {
+    const maxFiles = opts.maxFiles ?? 11;
+    const maxUrls = opts.maxUrls ?? 50_000;
+    const sources = [];
+    const entries = new Map();
+    let truncated = false;
+    let fetched = 0;
+    const fetchOne = async (url) => {
+        if (/\.gz($|\?)/i.test(url)) {
+            sources.push({ url, status: -1, kind: "gzip-skipped", count: 0 });
+            return {};
+        }
+        fetched++;
+        const res = await fetcher(url, {
+            redirect: "follow",
+            maxBytes: 25_000_000,
+            headers: { accept: "application/xml,text/xml,text/plain,*/*" },
+        });
+        if (res.status !== 200) {
+            sources.push({ url, status: res.status, kind: "error", count: 0, error: res.error });
+            return { res };
+        }
+        if (looksLikeHtml(res.body, res.headers["content-type"])) {
+            sources.push({ url, status: res.status, kind: "html", count: 0, error: "returned an HTML page" });
+            return { res };
+        }
+        const parsed = parseSitemapXml(res.body);
+        sources.push({ url, status: res.status, kind: parsed.kind, count: parsed.kind === "index" ? parsed.children.length : parsed.entries.length });
+        return { res, parsed };
+    };
+    const queue = [];
+    if (declared.length) {
+        queue.push(...[...new Set(declared)].slice(0, 5));
+    }
+    else {
+        for (const cand of ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml"]) {
+            const { parsed } = await fetchOne(origin + cand);
+            if (parsed && parsed.kind !== "invalid") {
+                absorb(parsed);
+                break;
+            }
+        }
+    }
+    function absorb(parsed) {
+        if (parsed.kind === "index")
+            for (const c of parsed.children)
+                queue.push(c.loc);
+        for (const e of parsed.entries) {
+            if (entries.size >= maxUrls) {
+                truncated = true;
+                break;
+            }
+            if (!entries.has(e.loc))
+                entries.set(e.loc, e);
+        }
+    }
+    const seen = new Set(sources.map((s) => s.url));
+    while (queue.length && fetched < maxFiles && entries.size < maxUrls) {
+        const next = queue.shift();
+        if (seen.has(next))
+            continue;
+        seen.add(next);
+        const { parsed } = await fetchOne(next);
+        if (parsed)
+            absorb(parsed);
+    }
+    if (queue.length)
+        truncated = true;
+    return { sources, entries: [...entries.values()], declared: declared.length > 0, truncated };
+}
+export function lastmodStats(entries, now = new Date()) {
+    const days = new Map();
+    let withLastmod = 0;
+    let future = 0;
+    const today = now.toISOString().slice(0, 10);
+    let todayCount = 0;
+    for (const e of entries) {
+        if (!e.lastmod)
+            continue;
+        const d = e.lastmod.slice(0, 10);
+        if (!/^\d{4}-\d{2}-\d{2}$/.test(d))
+            continue;
+        withLastmod++;
+        days.set(d, (days.get(d) ?? 0) + 1);
+        if (d === today)
+            todayCount++;
+        const t = Date.parse(e.lastmod);
+        if (Number.isFinite(t) && t > now.getTime() + 36 * 3600 * 1000)
+            future++;
+    }
+    let commonDay;
+    let max = 0;
+    for (const [d, n] of days)
+        if (n > max)
+            [commonDay, max] = [d, n];
+    return {
+        total: entries.length,
+        withLastmod,
+        distinctDays: days.size,
+        identical: withLastmod >= 5 && days.size === 1,
+        stampedToday: withLastmod >= 5 && todayCount / withLastmod >= 0.9,
+        future,
+        commonDay,
+    };
+}

package/dist/tools/crawl/cli.js

@@ -0,0 +1,106 @@
+// CLI commands for the crawl module: scan, audit, indexnow, keywords. They print the same markdown as the MCP tools.
+import path from "node:path";
+import { parseArgs } from "../../cli.js";
+import { loadConfig, loadResearch, logEvent, readJson, saveResearch } from "../../state.js";
+import { auditAndSave, scanAndSave } from "./index.js";
+import { indexNowMarkdown, runIndexNow } from "./indexnow.js";
+import { keywordsMarkdown, mergeKeywords, runKeywords, termsToSave } from "./keywords.js";
+const str = (v) => (typeof v === "string" ? v : undefined);
+const list = (v) => typeof v === "string" ? v.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
+function fail(msg) {
+    console.error(msg);
+    return 1;
+}
+export const commands = {
+    async scan(args) {
+        const a = parseArgs(args);
+        const root = path.resolve(a._[0] ?? str(a.flags.root) ?? process.cwd());
+        const max = Number(str(a.flags["max-files"]) ?? NaN);
+        try {
+            const { markdown } = await scanAndSave(root, Number.isFinite(max) ? max : undefined, !a.flags["no-save"]);
+            console.log(markdown);
+            return 0;
+        }
+        catch (e) {
+            return fail(`scan failed: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    },
+    async audit(args) {
+        const a = parseArgs(args);
+        const root = path.resolve(str(a.flags.root) ?? process.cwd());
+        const max = Number(str(a.flags.max) ?? NaN);
+        try {
+            const { run, markdown } = await auditAndSave(root, {
+                site_url: a._[0] ?? str(a.flags.site),
+                max_urls: Number.isFinite(max) ? max : undefined,
+                urls: list(a.flags.urls),
+                include_bot_test: a.flags["no-bots"] ? false : undefined,
+                preview_urls: list(a.flags.preview),
+            }, (done, total, message) => process.stderr.write(`\r[${done}/${total}] ${message}                    `), !a.flags["no-save"]);
+            process.stderr.write("\n");
+            console.log(markdown);
+            // --strict: non-zero exit when critical findings exist (for CI / a pre-publish script, book P14)
+            if (a.flags.strict && run.findings.some((f) => f.severity === "critical"))
+                return 2;
+            return 0;
+        }
+        catch (e) {
+            process.stderr.write("\n");
+            return fail(`audit failed: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    },
+    async indexnow(args) {
+        const a = parseArgs(args);
+        const root = path.resolve(str(a.flags.root) ?? process.cwd());
+        try {
+            const config = await loadConfig(root);
+            const research = await loadResearch(root);
+            const siteUrl = str(a.flags.site) ?? a._[0] ?? config.siteUrl ?? research?.site?.url;
+            if (!siteUrl)
+                return fail("No site URL: pass --site https://example.com (or run `katman-mcp audit <url>` first).");
+            const scan = await readJson(root, "scan.json");
+            const run = await runIndexNow({ root, siteUrl, urls: list(a.flags.urls), key: str(a.flags.key), dryRun: !a.flags.submit, staticDir: scan?.staticDir });
+            await logEvent(root, `katman indexnow ${run.host}: ${run.dryRun ? "dry run" : run.submitted ? `HTTP ${run.submitted.status}` : "nothing sent"}`);
+            console.log(indexNowMarkdown(run));
+            return run.submitted && run.submitted.status !== 200 && run.submitted.status !== 202 ? 1 : 0;
+        }
+        catch (e) {
+            return fail(`indexnow failed: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    },
+    async keywords(args) {
+        const a = parseArgs(args);
+        const seed = a._.join(" ").trim();
+        if (!seed)
+            return fail('Usage: katman-mcp keywords "<seed>" [--lang en] [--country us] [--no-expand] [--save] [--root .]');
+        try {
+            const r = await runKeywords({ seed, lang: str(a.flags.lang), country: str(a.flags.country), expand: !a.flags["no-expand"] });
+            let saved;
+            if (a.flags.save) {
+                const root = path.resolve(str(a.flags.root) ?? process.cwd());
+                const research = await loadResearch(root);
+                if (!research)
+                    saved = { added: 0, note: "Not saved: no .katman/research.json (run katman_research first)." };
+                else {
+                    const added = mergeKeywords(research, termsToSave(r));
+                    await saveResearch(root, research);
+                    saved = { added };
+                }
+            }
+            console.log(keywordsMarkdown(r, saved));
+            return 0;
+        }
+        catch (e) {
+            return fail(`keywords failed: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    },
+};
+export const help = [
+    "  katman-mcp scan [path] [--no-save]          scan the codebase → .katman/scan.json",
+    "  katman-mcp audit <url> [--max 50] [--root .] [--urls u1,u2] [--preview u1] [--no-bots] [--strict] [--no-save]",
+    "                                              audit the live site like AI crawlers see it (--strict: exit 2 on critical)",
+    "  katman-mcp indexnow [--root .] [--site url] [--urls u1,u2] [--submit]",
+    "                                              IndexNow key check + new/changed URLs (dry run unless --submit)",
+    '  katman-mcp keywords "<seed>" [--lang en] [--country us] [--no-expand] [--save]',
+    "                                              Google autocomplete ideas (≤40 requests)",
+];

package/dist/tools/crawl/common/advice.js

@@ -0,0 +1,61 @@
+// Fix texts shared by scan and audit findings. Sources: the Katman book (ch. 5, 6, 12, 13) and
+// research/tech-web-indexing.md (Cloudflare's 2026-07-01 / 2026-09-15 AI-bot policy changes).
+export const JS_RENDER_NOTE = "No AI crawler runs JavaScript (only Googlebot, Bingbot and Applebot render pages), so a client-rendered page can appear in Google, Bing, Copilot and Gemini while looking empty to ChatGPT, Claude and Perplexity.";
+export const LOVABLE_CAVEAT = "Lovable may serve prerendered HTML to verified bots only; confirm with Search Console URL Inspection (Test live URL → View crawled page). Lovable apps created since 2026-05-13 are server-rendered (TanStack Start); older ones can use \"upgrade to TanStack Start\".";
+export const CSR_ACCEPT = "JavaScript disabled, every public page's H1 and first paragraph are readable";
+export const CLOUDFLARE_STEPS = [
+    "Cloudflare dashboard → your domain → Security → Settings → Configure AI bot policies: Search = Allow, Agent = Allow.",
+    'Training is your business decision. To opt out, choose "Disallow AI Training" (a no-training signal that keeps Googlebot, Bingbot and Applebot allowed). Never choose "Block" or "Block on pages with ads": since 15 Sep 2026 that also blocks Googlebot, Bingbot and Applebot and drops the site from Google and Bing.',
+    'Turn off the legacy "Block AI bots" toggle if it is still shown, Managed robots.txt (publish your own, book appendix A), AI Labyrinth, and Bot Fight Mode if it challenges bots. Turn off Email Address Obfuscation so assistants can read your email address as text.',
+    "Security → Analytics → Events shows which rule refused the request; AI Crawl Control → Metrics shows the status codes AI bots receive.",
+    'API: GET/PUT /zones/<zone_id>/bot_management with {"ai_search":"disabled","ai_user":"disabled","ai_training":"disallow","ai_bots_protection":"disabled","fight_mode":false,"is_robots_txt_managed":false} ("disabled" = not blocked; use "disabled" for ai_training too if you allow training).',
+    "Re-run katman_audit_site. OpenAI says changes take about 24 hours to reach ChatGPT search.",
+];
+export function cloudflareFix() {
+    return CLOUDFLARE_STEPS.map((s, i) => `(${i + 1}) ${s}`).join(" ");
+}
+export function genericCdnFix(cdn) {
+    switch (cdn) {
+        case "vercel":
+            return "Vercel → Project → Firewall: check the Bot Management / AI Bots ruleset (it should be Allow or Log, not Deny) and any custom rules matching these user agents; Attack Challenge Mode also challenges bots.";
+        case "netlify":
+            return "Netlify: remove the User Agent Blocker extension rules (or any edge function) that match these user agents.";
+        case "cloudfront":
+            return "AWS: check AWS WAF Bot Control / managed rule groups on the CloudFront distribution; allow verified AI search and user bots.";
+        case "akamai":
+            return "Akamai: check Bot Manager categories for AI crawlers and search engines; allow them.";
+        default:
+            return "Check the CDN/WAF/firewall in front of the site (bot protection, rate limits, user-agent block lists) and allow the AI search and user bots; make it say the same thing as robots.txt.";
+    }
+}
+/** Framework-specific way to put page text into the server HTML (P2). */
+export function csrFixFor(framework, router) {
+    switch (framework) {
+        case "vite-react":
+        case "cra":
+            return "Pre-render the public routes at build time (vite-react-ssg, or vite-prerender-plugin with react-dom/server; react-snap is unmaintained), or move the public pages to an SSR/SSG framework (TanStack Start, React Router framework mode with prerender, Next.js, Astro). Keep the logged-in app client-side. Use real paths (BrowserRouter), not #.";
+        case "vite-vue":
+            return "Pre-render the public routes with vite-ssg, or move the marketing pages to Nuxt (SSR/SSG). Keep the app screens client-side.";
+        case "vite-svelte":
+            return "Move the public pages to SvelteKit with `export const prerender = true`, or prerender them at build time.";
+        case "vite-solid":
+            return "Move the public pages to SolidStart (SSR or prerender).";
+        case "angular":
+            return "Add @angular/ssr (ng add @angular/ssr) and prerender the public routes.";
+        case "nuxt":
+            return "Remove `ssr: false` from nuxt.config (or set routeRules with prerender: true for public pages).";
+        case "sveltekit":
+            return "Remove `export const ssr = false` from the root +layout, and set `export const prerender = true` on public routes (or use adapter-static without the SPA fallback).";
+        case "remix":
+        case "react-router":
+            return "Turn SPA mode off (ssr: true) or list the public routes in `prerender` in react-router.config.ts.";
+        case "solid-start":
+            return "Remove `ssr: false` from app.config.";
+        case "next":
+            return router === "pages"
+                ? "Render the page text on the server (getStaticProps/getServerSideProps) instead of fetching it in useEffect; avoid dynamic(..., { ssr: false }) for content."
+                : "Fetch the content in a Server Component (or generateStaticParams + static rendering) instead of useEffect in a \"use client\" page; avoid dynamic(..., { ssr: false }) for content.";
+        default:
+            return "Render the public pages' text on the server (SSR), at build time (SSG) or with a prerender step; keep the logged-in app as it is.";
+    }
+}

package/dist/tools/crawl/common/findings.js

@@ -0,0 +1,57 @@
+const SEV = { critical: 100, high: 45, medium: 18, low: 6, info: 1 };
+const GATE = { read: 1.5, quote: 1.2, fit: 1.15, measure: 0.8, mention: 0.7 };
+/**
+ * Traffic-impact score. Severity dominates; within a severity, the read gate (nothing else matters if bots can't
+ * read the page) outranks quote, which outranks fit, and findings touching many pages or the homepage rank higher.
+ */
+export function impactScore(f, ctx) {
+    const n = f.urls?.length ?? 0;
+    const breadth = ctx.pages > 0 ? Math.min(1, n / Math.max(3, ctx.pages)) : 0;
+    const home = ctx.homeUrl && f.urls?.some((u) => u === ctx.homeUrl) ? 8 : 0;
+    const siteWide = /^(site|robots|bots|sitemap|https|host|soft-404|compression|cdn)/.test(f.id) ? 6 : 0;
+    return SEV[f.severity] * GATE[f.gate] * (1 + breadth * 0.6) + home + siteWide;
+}
+export function rankByImpact(fs, ctx) {
+    return [...fs].sort((a, b) => impactScore(b, ctx) - impactScore(a, ctx));
+}
+/** Markdown list in the given order (util.findingsToMarkdown re-sorts by severity). */
+export function findingsList(fs, opts = {}) {
+    const limit = opts.limit ?? fs.length;
+    if (!fs.length)
+        return "No issues found.";
+    return fs
+        .slice(0, limit)
+        .map((f, i) => {
+        const lines = [
+            `${i + 1}. **[${f.severity.toUpperCase()}] ${f.title}** (gate: ${f.gate}${f.prompt ? ` · fix with ${f.prompt}` : ""})`,
+            `   ${f.detail}`,
+        ];
+        if (f.evidence?.length)
+            lines.push(`   Evidence: ${f.evidence.slice(0, 6).join(" · ")}`);
+        if (f.files?.length)
+            lines.push(`   Files: ${f.files.slice(0, 8).join(", ")}`);
+        if (f.urls?.length) {
+            const n = opts.urlLimit ?? 8;
+            lines.push(`   URLs: ${f.urls.slice(0, n).join(", ")}${f.urls.length > n ? ` …(+${f.urls.length - n})` : ""}`);
+        }
+        if (f.fix)
+            lines.push(`   Fix: ${f.fix}`);
+        if (f.accept)
+            lines.push(`   Accept when: ${f.accept}`);
+        return lines.join("\n");
+    })
+        .join("\n");
+}
+export function countBySeverity(fs) {
+    const out = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
+    for (const f of fs)
+        out[f.severity]++;
+    return out;
+}
+export function severityLine(fs) {
+    const c = countBySeverity(fs);
+    return ["critical", "high", "medium", "low", "info"]
+        .filter((s) => c[s])
+        .map((s) => `${c[s]} ${s}`)
+        .join(" · ") || "no findings";
+}

package/dist/tools/crawl/common/routes.js

@@ -0,0 +1,80 @@
+import { fold } from "./text.js";
+const API = new Set(["api", "graphql", "trpc", "webhook", "webhooks", "_next", "cdn-cgi", "rpc", "_vercel", "__data.json"]);
+const AUTH = new Set([
+    "login", "log-in", "signin", "sign-in", "signup", "sign-up", "register", "logout", "log-out", "signout", "sign-out",
+    "forgot", "forgot-password", "reset", "reset-password", "verify", "verify-email", "auth", "oauth", "callback", "sso",
+    "magic-link", "password", "invite", "invited", "accept-invite",
+    // tr / es / de / fr / it / pt
+    "giris", "giris-yap", "kayit", "kayit-ol", "uye-ol", "cikis", "sifremi-unuttum", "sifre-sifirla",
+    "iniciar-sesion", "registro", "registrarse", "olvide-mi-contrasena", "anmelden", "registrieren", "abmelden",
+    "passwort-vergessen", "connexion", "inscription", "deconnexion", "mot-de-passe-oublie", "accedi", "registrati",
+    "password-dimenticata", "entrar", "cadastro", "sair", "esqueci-a-senha",
+]);
+const CHECKOUT = new Set([
+    "checkout", "cart", "basket", "payment", "payments", "pay", "thank-you", "thankyou", "thanks", "success",
+    "order-confirmation", "purchase", "order", "orders", "sepet", "odeme", "tesekkurler", "danke", "gracias", "merci",
+    "grazie", "obrigado", "kasse", "paiement", "pago", "billing-success",
+]);
+const APP = new Set([
+    "dashboard", "app", "account", "accounts", "settings", "profile", "admin", "billing", "workspace", "workspaces",
+    "console", "inbox", "notifications", "onboarding", "portal", "my-account", "me", "hesabim", "hesap", "ayarlar",
+    "konto", "compte", "mi-cuenta", "minha-conta", "il-mio-account", "panel", "admin-panel",
+]);
+const SEARCH = new Set(["search", "arama", "buscar", "busqueda", "suche", "recherche", "cerca", "pesquisa", "results"]);
+const LEGAL = new Set([
+    "legal", "privacy", "privacy-policy", "terms", "terms-of-service", "terms-of-use", "terms-and-conditions", "tos",
+    "cookies", "cookie", "cookie-policy", "imprint", "impressum", "gdpr", "dpa", "disclaimer", "acceptable-use", "aup",
+    "refund", "refunds", "refund-policy", "kvkk", "gizlilik", "gizlilik-politikasi", "kullanim-kosullari",
+    "cerez-politikasi", "cerezler", "aydinlatma-metni", "mesafeli-satis-sozlesmesi", "datenschutz", "agb",
+    "mentions-legales", "confidentialite", "privacidad", "terminos", "aviso-legal", "privacy-notice", "copyright",
+    "dmca", "licenses", "usage-rights",
+]);
+/** Classify a URL path (or route pattern) by its segments. */
+export function classifyPath(pathOrUrl) {
+    let p = pathOrUrl;
+    const q = p.indexOf("?");
+    let query = "";
+    if (q >= 0) {
+        query = p.slice(q + 1);
+        p = p.slice(0, q);
+    }
+    if (/^https?:\/\//i.test(p)) {
+        try {
+            p = new URL(p).pathname;
+        }
+        catch {
+            /* keep */
+        }
+    }
+    const segs = p
+        .split("/")
+        .filter(Boolean)
+        .map((s) => fold(decodeSafe(s)).replace(/^[:$[(]+|[\])?*]+$/g, ""));
+    if (segs[0] && API.has(segs[0]))
+        return "api";
+    if (segs.some((s) => API.has(s) && s !== "rpc"))
+        return "api";
+    if (segs.some((s) => AUTH.has(s)))
+        return "auth";
+    if (segs.some((s) => CHECKOUT.has(s)))
+        return "checkout";
+    if (segs.some((s) => APP.has(s)))
+        return "app";
+    if (segs.some((s) => SEARCH.has(s)) || /(^|&)(q|query|s|search)=/.test(query))
+        return "search";
+    if (segs.some((s) => LEGAL.has(s)))
+        return "legal";
+    return "public";
+}
+function decodeSafe(s) {
+    try {
+        return decodeURIComponent(s);
+    }
+    catch {
+        return s;
+    }
+}
+/** Kinds whose pages are public content (count toward the gates). */
+export function isContentKind(k) {
+    return k === "public" || k === "legal" || k === "unknown";
+}