katman-mcp 0.1.0

package/dist/tools/crawl/common/text.js

@@ -0,0 +1,89 @@
+// Text helpers shared by the scan and audit tools. Pure functions, no I/O.
+const NAMED = {
+    amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", ndash: "–", mdash: "—", hellip: "…",
+    rsquo: "’", lsquo: "‘", ldquo: "“", rdquo: "”", copy: "©", reg: "®", trade: "™", middot: "·", bull: "•",
+};
+export function decodeEntities(s) {
+    return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
+        if (e[0] === "#") {
+            const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+            return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
+        }
+        return NAMED[e.toLowerCase()] ?? m;
+    });
+}
+/** Collapse whitespace (incl. nbsp) and trim. */
+export function collapse(s) {
+    return s.replace(/[\s ]+/g, " ").trim();
+}
+/** Words = whitespace-separated tokens containing a letter or digit (same idea as `sed 's/<[^>]*>/ /g' | wc -w`). */
+export function countWords(s) {
+    let n = 0;
+    for (const t of s.split(/[\s ]+/))
+        if (/[\p{L}\p{N}]/u.test(t))
+            n++;
+    return n;
+}
+/** Lowercase and strip diacritics (Turkish dotless ı → i) for matching. */
+export function fold(s) {
+    return s
+        .toLocaleLowerCase("en")
+        .replace(/ı/g, "i")
+        .normalize("NFKD")
+        .replace(/[̀-ͯ]/g, "");
+}
+const STOP = new Set((
+// English
+"the and for with your our you are from that this into over under about have has had was were will would can could " +
+    "not but all any its it's their them they what when where which who why how than then there these those been being " +
+    "also just more most very such only own same other each both few some out off per via get got use used using " +
+    // Turkish
+    "ile icin bir bu da de ve veya ama gibi daha cok en her hic mi mu ne ki olan olarak ise sonra once kadar diye " +
+    "size sizin bizim onun benim senin").split(/\s+/));
+/** Content tokens (≥3 chars, folded, no stopwords). */
+export function contentTokens(s) {
+    return fold(s)
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter((t) => t.length >= 3 && !STOP.has(t));
+}
+function tokenHit(t, set, list) {
+    if (set.has(t))
+        return true;
+    // Tolerate suffixes (plural, Turkish case endings): shared prefix of ≥5 chars.
+    if (t.length >= 5) {
+        const stem = t.slice(0, Math.max(5, t.length - 3));
+        return list.some((o) => o.length >= 5 && (o.startsWith(stem) || t.startsWith(o.slice(0, Math.max(5, o.length - 3)))));
+    }
+    return false;
+}
+/** Share of `needle`'s content tokens that also appear in `hay` (0–1). */
+export function containment(needle, hay) {
+    const a = [...new Set(contentTokens(needle))];
+    if (!a.length)
+        return 1;
+    const bl = [...new Set(contentTokens(hay))];
+    const bs = new Set(bl);
+    let hit = 0;
+    for (const t of a)
+        if (tokenHit(t, bs, bl))
+            hit++;
+    return hit / a.length;
+}
+export function pct(n, d) {
+    return d ? `${Math.round((100 * n) / d)}%` : "–";
+}
+export function plural(n, one, many = one + "s") {
+    return `${n} ${n === 1 ? one : many}`;
+}
+export function slugify(s) {
+    return fold(s).replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "x";
+}
+export function escapeRe(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/** First n items + "…and N more". */
+export function listSome(items, n = 5) {
+    if (items.length <= n)
+        return items.join(", ");
+    return `${items.slice(0, n).join(", ")} …and ${items.length - n} more`;
+}

package/dist/tools/crawl/common/urls.js

@@ -0,0 +1,64 @@
+// URL helpers shared by the audit and IndexNow tools.
+export function safeUrl(u, base) {
+    try {
+        return base ? new URL(u, base) : new URL(u);
+    }
+    catch {
+        return undefined;
+    }
+}
+export function stripWww(host) {
+    return host.toLowerCase().replace(/^www\./, "");
+}
+/** Same site = same host ignoring a leading www. */
+export function sameSite(hostA, hostB) {
+    return stripWww(hostA) === stripWww(hostB);
+}
+const TRACKING = /^(utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|ref|ref_src)$/i;
+/**
+ * Stable key for "the same page": host without www, path without trailing slash, query minus tracking params,
+ * no fragment, protocol ignored. Used to match links, sitemap entries and crawled pages.
+ */
+export function urlKey(input) {
+    const u = typeof input === "string" ? safeUrl(input) : input;
+    if (!u)
+        return String(input);
+    let p = u.pathname || "/";
+    if (p.length > 1)
+        p = p.replace(/\/+$/, "") || "/";
+    const params = [...u.searchParams.entries()].filter(([k]) => !TRACKING.test(k));
+    params.sort(([a], [b]) => a.localeCompare(b));
+    const q = params.length ? "?" + params.map(([k, v]) => `${k}=${v}`).join("&") : "";
+    return `${stripWww(u.hostname)}${u.port ? ":" + u.port : ""}${p}${q}`;
+}
+/** Exact-URL comparison used for "canonical points to itself": scheme + host + path + query, no fragment. */
+export function sameUrlExact(a, b) {
+    const x = safeUrl(a);
+    const y = safeUrl(b);
+    if (!x || !y)
+        return false;
+    x.hash = "";
+    y.hash = "";
+    return x.href === y.href;
+}
+const ASSET_EXT = /\.(png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|pdf|zip|gz|tgz|tar|rar|7z|mp3|mp4|m4a|wav|ogg|oga|webm|mov|avi|mkv|flac|css|js|mjs|json|xml|txt|csv|xlsx?|docx?|pptx?|woff2?|ttf|otf|eot|map|rss|atom|dmg|exe|apk|ipa|wasm|webmanifest)$/i;
+export function isAssetPath(pathname) {
+    return ASSET_EXT.test(pathname);
+}
+export function originOf(u) {
+    const x = safeUrl(u);
+    return x ? x.origin : u;
+}
+export function pathOf(u) {
+    const x = safeUrl(u);
+    return x ? x.pathname + x.search : u;
+}
+/** Shorten a URL for tables: path only when it is on the audited host. */
+export function shortUrl(u, host) {
+    const x = safeUrl(u);
+    if (!x)
+        return u;
+    if (sameSite(x.hostname, host))
+        return (x.pathname || "/") + x.search;
+    return x.href;
+}

package/dist/tools/crawl/index.js

@@ -0,0 +1,226 @@
+// Crawl module: katman_scan_code, katman_audit_site, katman_indexnow, katman_keywords.
+// Registration only; the engines live in scan/, audit/, indexnow.ts and keywords.ts (pure or fetcher-injected).
+import { stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { z } from "zod";
+import { resolveRoot, rootWarning } from "../../context.js";
+import { loadConfig, loadResearch, logEvent, readJson, saveConfig, saveResearch, stamp, writeJson, writeText } from "../../state.js";
+import { fail, ok } from "../../util.js";
+import { PRO_NOTE, withPro } from "../../license.js";
+import { topIssues, auditMarkdown } from "./audit/report.js";
+import { runAudit } from "./audit/index.js";
+import { countBySeverity } from "./common/findings.js";
+import { indexNowMarkdown, runIndexNow } from "./indexnow.js";
+import { keywordsMarkdown, mergeKeywords, runKeywords, termsToSave } from "./keywords.js";
+import { runScan, scanMarkdown } from "./scan/index.js";
+function unsafeRoot(root) {
+    return root === path.parse(root).root || root === os.homedir();
+}
+function errText(e) {
+    return e instanceof Error ? e.message : String(e);
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Shared runners (used by the MCP tools and the CLI)
+export async function scanAndSave(root, maxFiles, save = true) {
+    const st = await stat(root).catch(() => undefined);
+    if (!st?.isDirectory())
+        throw new Error(`${root} is not a directory: pass project_root with the absolute path of the website's codebase.`);
+    const warning = rootWarning(root);
+    const run = await runScan(root, { maxFiles });
+    if (save) {
+        await writeJson(root, "scan.json", run.result);
+        await logEvent(root, `katman_scan_code: ${run.result.framework.name} · ${run.result.rendering.verdict} · ${run.result.findings.length} findings`);
+    }
+    const markdown = scanMarkdown(run, { savedTo: save ? ".katman/scan.json" : undefined, warning });
+    return { run, markdown };
+}
+export async function auditAndSave(root, a, onProgress, save = true) {
+    const writable = save && !unsafeRoot(root);
+    const readable = !unsafeRoot(root);
+    const config = readable ? await loadConfig(root) : {};
+    const research = readable ? await loadResearch(root) : undefined;
+    const scan = readable ? await readJson(root, "scan.json") : undefined;
+    const siteUrl = a.site_url?.trim() || config.siteUrl || research?.site?.url;
+    if (!siteUrl)
+        throw new Error("No site URL: pass site_url (e.g. https://example.com), or save it first with katman_start { site_url }.");
+    const run = await runAudit({
+        siteUrl,
+        maxUrls: a.max_urls ?? config.maxUrls ?? 50,
+        urls: a.urls,
+        includeBotTest: a.include_bot_test,
+        previewUrls: a.preview_urls,
+        research,
+        scan,
+        onProgress,
+    });
+    let jsonPath;
+    let mdPath;
+    if (writable) {
+        const rel = `audits/${stamp(new Date(run.result.ts))}.json`;
+        await writeJson(root, rel, run.result);
+        jsonPath = `.katman/${rel}`;
+        mdPath = ".katman/audit.md";
+        await writeText(root, "audit.md", auditMarkdown(run.ctx, run.result, run.findings, { full: true, jsonPath, mdPath }) + "\n");
+        if (!config.siteUrl && run.ctx.home && run.ctx.home.status > 0) {
+            config.siteUrl = run.ctx.origin;
+            await saveConfig(root, config);
+        }
+        const g = run.result.gates;
+        await logEvent(root, `katman_audit_site ${run.ctx.origin}: ${run.ctx.pages.length} URLs · read ${g.read.pass}/${g.read.total} · quote ${g.quote.pass}/${g.quote.total} · fit ${g.fit.pass}/${g.fit.total} · ${run.findings.length} findings`);
+    }
+    const markdown = auditMarkdown(run.ctx, run.result, run.findings, { full: false, jsonPath, mdPath });
+    return { run, markdown, jsonPath, writable };
+}
+function auditSummary(result, top, jsonPath) {
+    return {
+        siteUrl: result.siteUrl,
+        pages: result.pages.length,
+        gates: Object.fromEntries(Object.entries(result.gates).map(([k, v]) => [k, { pass: v.pass, total: v.total }])),
+        severities: countBySeverity(result.findings),
+        topIssues: top.map((f) => ({ id: f.id, severity: f.severity, gate: f.gate, prompt: f.prompt, title: f.title })),
+        bots: result.site.bots.filter((b) => b.blocked).map((b) => b.token),
+        auditFile: jsonPath,
+    };
+}
+// ---------------------------------------------------------------------------------------------------------------
+export function registerCrawlTools(server, ctx) {
+    server.registerTool("katman_scan_code", {
+        title: "Scan the codebase (framework, rendering, SEO files)",
+        description: "Read-only scan of the project's code (no network). Detects framework and builder (Lovable/Bolt/v0/Replit), the rendering verdict (SSR/SSG/CSR + evidence), hosting and preview-URL patterns, public vs app/auth routes, robots/sitemap/llms.txt/404 (static or generated), head-tag source, JSON-LD, i18n, analytics/AI-channel tracking, onClick navigation, hash routing and images without alt. Never opens .env*, keys or secret files. Writes .katman/scan.json; returns findings with gate, P-code and acceptance criterion. Run it first, and after big code changes.",
+        inputSchema: {
+            project_root: z.string().optional().describe("Absolute path of the website's codebase (default: KATMAN_PROJECT_ROOT, MCP roots, or cwd)."),
+            max_files: z.number().int().min(50).max(50000).optional().describe("Max files to list (default 5000)."),
+        },
+        annotations: { readOnlyHint: true, openWorldHint: false, destructiveHint: false, idempotentHint: true },
+    }, async ({ project_root, max_files }) => {
+        try {
+            const root = await resolveRoot(ctx, project_root);
+            if (unsafeRoot(root))
+                return fail(rootWarning(root) ?? `Refusing to scan ${root}: pass project_root.`);
+            const { run, markdown } = await scanAndSave(root, max_files);
+            const r = run.result;
+            return ok(markdown, {
+                scanFile: ".katman/scan.json",
+                framework: r.framework,
+                rendering: r.rendering,
+                hosting: r.hosting,
+                staticDir: r.staticDir,
+                files: r.files,
+                routes: r.routes.length,
+                severities: countBySeverity(r.findings),
+                findings: r.findings.map((f) => ({ id: f.id, severity: f.severity, gate: f.gate, prompt: f.prompt, title: f.title })),
+            });
+        }
+        catch (e) {
+            return fail(`katman_scan_code failed: ${errText(e)}`);
+        }
+    });
+    server.registerTool("katman_audit_site", {
+        title: "Audit the live site like AI crawlers see it",
+        description: "Fetches the live site without JavaScript, like AI crawlers: robots.txt, sitemap, http→https, www/apex, HSTS, compression, soft 404, llms.txt, AI-bot access through the CDN/firewall (GPTBot, OAI-SearchBot, ClaudeBot, PerplexityBot…), and per page: status, noindex, title/description, canonical, H1, JS-off words and first paragraph, JSON-LD, og:image, hreflang, internal links. Returns gates, top-10 issues (P-codes + acceptance criteria) and the P14 table. Network: the site only, ≤4 requests at a time. Pass urls to re-verify a fix. Writes .katman/audits/<stamp>.json + .katman/audit.md.",
+        inputSchema: {
+            site_url: z.string().optional().describe("Site to audit, e.g. https://example.com (default: .katman/config.json siteUrl or research site.url)."),
+            project_root: z.string().optional().describe("Absolute project path (where .katman/ lives)."),
+            max_urls: z.number().int().min(1).max(500).optional().describe("Max pages to fetch: homepage, its links and sitemap URLs (default 50)."),
+            urls: z.array(z.string()).optional().describe("Quick verify mode: audit only these URLs (+ homepage and site-level checks)."),
+            include_bot_test: z.boolean().optional().describe("Test the homepage with each AI crawler user agent (default true)."),
+            preview_urls: z.array(z.string()).optional().describe("Preview/staging copies (e.g. https://proj.vercel.app) that must 301 to the main host or send noindex."),
+        },
+        annotations: { readOnlyHint: true, openWorldHint: true, destructiveHint: false, idempotentHint: true },
+    }, async (args, extra) => {
+        try {
+            const root = await resolveRoot(ctx, args.project_root);
+            const token = extra?._meta?.progressToken;
+            const onProgress = token !== undefined
+                ? (done, total, message) => {
+                    void extra.sendNotification({ method: "notifications/progress", params: { progressToken: token, progress: done, total, message } }).catch(() => undefined);
+                }
+                : undefined;
+            const { run, markdown, jsonPath, writable } = await auditAndSave(root, args, onProgress);
+            const note = writable ? "" : `\n\n> Not saved: ${root} is not a project folder (pass project_root to keep the audit in .katman/).`;
+            return ok(markdown + note, auditSummary(run.result, topIssues(run.ctx, run.findings), jsonPath));
+        }
+        catch (e) {
+            return fail(`katman_audit_site failed: ${errText(e)}`);
+        }
+    });
+    server.registerTool("katman_indexnow", {
+        title: "Notify search engines of new/changed pages (IndexNow)",
+        description: "Tells Bing (→ Copilot, ChatGPT search, DuckDuckGo), Yandex, Seznam, Naver and Yep about new or changed URLs via IndexNow. Creates a 32-hex key (saved in .katman/config.json), checks https://<host>/<key>.txt serves it, and selects only new/changed URLs (sitemap lastmod, else a hash of the page text) against .katman/indexnow.json. dry_run (default true) sends nothing; dry_run:false POSTs to api.indexnow.org and records the URLs. Always returns the manual checklist for Google Search Console, Bing Webmaster Tools and Brave." + PRO_NOTE,
+        inputSchema: {
+            project_root: z.string().optional().describe("Absolute project path (where .katman/ lives)."),
+            site_url: z.string().optional().describe("Site, e.g. https://example.com (default: config siteUrl)."),
+            urls: z.array(z.string()).optional().describe("Submit exactly these URLs instead of diffing the sitemap."),
+            key: z.string().optional().describe("Existing IndexNow key (8–128 chars a-z A-Z 0-9 -); default: config or a new one."),
+            dry_run: z.boolean().optional().describe("Default true: plan only. false = submit to api.indexnow.org."),
+        },
+        annotations: { readOnlyHint: false, openWorldHint: true, destructiveHint: false, idempotentHint: false },
+    }, withPro(ctx, "indexnow", async ({ project_root, site_url, urls, key, dry_run }) => {
+        try {
+            const root = await resolveRoot(ctx, project_root);
+            if (unsafeRoot(root))
+                return fail(`Pass project_root: ${root} is not a project folder (the key and records are saved in .katman/).`);
+            const config = await loadConfig(root);
+            const research = await loadResearch(root);
+            const siteUrl = site_url?.trim() || config.siteUrl || research?.site?.url;
+            if (!siteUrl)
+                return fail("No site URL: pass site_url (e.g. https://example.com).");
+            const scan = await readJson(root, "scan.json");
+            const run = await runIndexNow({ root, siteUrl, urls, key, dryRun: dry_run !== false, staticDir: scan?.staticDir });
+            await logEvent(root, `katman_indexnow ${run.host}: ${run.dryRun ? "dry run" : run.submitted ? `HTTP ${run.submitted.status}` : "nothing sent"} · ${run.candidates.length} new/changed`);
+            return ok(indexNowMarkdown(run), {
+                host: run.host,
+                keyFileOk: run.keyFile.ok,
+                keyLocation: run.keyLocation,
+                dryRun: run.dryRun,
+                toSubmit: run.candidates.length,
+                unchanged: run.unchanged,
+                submitted: run.submitted,
+            });
+        }
+        catch (e) {
+            return fail(`katman_indexnow failed: ${errText(e)}`);
+        }
+    }));
+    server.registerTool("katman_keywords", {
+        title: "Google autocomplete keyword ideas",
+        description: "Expands a seed phrase with Google autocomplete (seed + a–z/0–9; ≤40 requests, 350 ms apart; network: suggestqueries.google.com only) into real search phrases, clustered, and marks single-question, situation and comparison page candidates (English and Turkish question words). Use during research to find the customer's own words. save:true merges them into .katman/research.json keywords (source \"autocomplete\"); otherwise nothing is written.",
+        inputSchema: {
+            seed: z.string().min(2).describe("Seed phrase in the customer's words, e.g. \"import invoices into accounting\"."),
+            lang: z.string().optional().describe("Interface language (hl), default en; tr uses the Turkish alphabet."),
+            country: z.string().optional().describe("Country (gl), default us."),
+            expand: z.boolean().optional().describe("Expand with letters/digits (default true); false = one request."),
+            save: z.boolean().optional().describe("Merge the candidates into research.json keywords."),
+            project_root: z.string().optional().describe("Absolute project path (needed with save)."),
+        },
+        annotations: { readOnlyHint: false, openWorldHint: true, destructiveHint: false, idempotentHint: true },
+    }, async ({ seed, lang, country, expand, save, project_root }) => {
+        try {
+            const r = await runKeywords({ seed, lang, country, expand });
+            let saved;
+            if (save) {
+                const root = await resolveRoot(ctx, project_root);
+                const research = unsafeRoot(root) ? undefined : await loadResearch(root);
+                if (!research)
+                    saved = { added: 0, note: "Not saved: no .katman/research.json yet — run katman_research (mode: template) first, then save again." };
+                else {
+                    const added = mergeKeywords(research, termsToSave(r));
+                    await saveResearch(root, research);
+                    await logEvent(root, `katman_keywords "${r.seed}": ${added} keywords saved`);
+                    saved = { added };
+                }
+            }
+            return ok(keywordsMarkdown(r, saved), {
+                seed: r.seed,
+                requests: r.requests,
+                suggestions: r.suggestions.length,
+                candidates: r.candidates,
+                clusters: r.clusters.slice(0, 20).map((c) => ({ key: c.key, count: c.items.length })),
+            });
+        }
+        catch (e) {
+            return fail(`katman_keywords failed: ${errText(e)}`);
+        }
+    });
+}

package/dist/tools/crawl/indexnow.js

@@ -0,0 +1,271 @@
+// katman_indexnow engine: key setup, key-file verification, "only new or changed URLs" selection (sitemap lastmod,
+// else a hash of the page's visible text), submission to api.indexnow.org, and the manual checklist for Google,
+// Bing and Brave (book ch. 13 + research/tech-web-indexing.md §3).
+import { createHash, randomBytes } from "node:crypto";
+import { httpGet, httpPostJson, mapLimit } from "../../net.js";
+import { loadConfig, readJson, saveConfig, writeJson } from "../../state.js";
+import { mdTable, normalizeSiteUrl } from "../../util.js";
+import { collapse, decodeEntities } from "./common/text.js";
+import { safeUrl } from "./common/urls.js";
+import { followRedirects } from "./audit/checks.js";
+import { analyzeRobots } from "./audit/robots.js";
+import { lastmodStats, loadSitemaps } from "./audit/sitemap.js";
+export const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
+const KEY_RE = /^[A-Za-z0-9-]{8,128}$/;
+export function generateKey() {
+    return randomBytes(16).toString("hex");
+}
+/** Hash of the page's visible text: markup, scripts and build hashes don't count as "changed". */
+export function textHash(html) {
+    const t = collapse(decodeEntities(html
+        .replace(/<(script|style|noscript|template|svg)\b[\s\S]*?<\/\1>/gi, " ")
+        .replace(/<!--[\s\S]*?-->/g, " ")
+        .replace(/<[^>]+>/g, " ")));
+    return createHash("sha256").update(t).digest("hex").slice(0, 32);
+}
+export function responseMessage(status) {
+    switch (status) {
+        case 200:
+            return "OK: URLs submitted and the key is valid.";
+        case 202:
+            return "Accepted: IndexNow received the URLs; key validation is pending (usually minutes).";
+        case 400:
+            return "Bad request: invalid format (every URL must be an absolute URL on the host).";
+        case 403:
+            return "Forbidden: the key is not valid — the key file was not found at keyLocation or its content doesn't match the key.";
+        case 422:
+            return "Unprocessable: the URLs don't belong to the host, or the key doesn't match the protocol's format.";
+        case 429:
+            return "Too many requests: IndexNow suspects spam. Send only new or changed URLs and wait before retrying.";
+        case 0:
+            return "No response from api.indexnow.org (network error or timeout).";
+        default:
+            return `Unexpected HTTP ${status} from api.indexnow.org.`;
+    }
+}
+export async function runIndexNow(o) {
+    const fetcher = o.fetcher ?? httpGet;
+    const post = o.post ?? ((u, d) => httpPostJson(u, d));
+    const now = o.now ?? new Date();
+    const dryRun = o.dryRun !== false;
+    const notes = [];
+    // Canonical origin (the key file and every URL must be on the same host)
+    let origin = normalizeSiteUrl(o.siteUrl);
+    const home = await followRedirects(origin + "/", fetcher, 5);
+    if (home.final.status > 0)
+        origin = new URL(home.chain[home.chain.length - 1].url).origin;
+    const host = new URL(origin).host;
+    // Key
+    const config = await loadConfig(o.root);
+    let key = (o.key ?? config.indexnowKey ?? "").trim();
+    let keyGenerated = false;
+    if (!key) {
+        key = generateKey();
+        keyGenerated = true;
+    }
+    if (!KEY_RE.test(key))
+        throw new Error(`Invalid IndexNow key "${key}": use 8–128 characters from a-z, A-Z, 0-9 and "-".`);
+    if (config.indexnowKey !== key) {
+        config.indexnowKey = key;
+        await saveConfig(o.root, config);
+    }
+    const keyLocation = `${origin}/${key}.txt`;
+    const kf = await fetcher(keyLocation, { redirect: "follow", headers: { accept: "text/plain,*/*" } });
+    const keyOk = kf.status === 200 && kf.body.trim() === key;
+    const keyFile = {
+        ok: keyOk,
+        status: kf.status,
+        detail: keyOk
+            ? "serves the key"
+            : kf.status === 200
+                ? /<html|<!doctype/i.test(kf.body)
+                    ? "answers with an HTML page (SPA fallback), not the key"
+                    : "answers 200 but the content is not exactly the key"
+                : kf.status === 0
+                    ? `no response (${kf.error ?? "error"})`
+                    : `HTTP ${kf.status}`,
+    };
+    // State
+    const stored = await readJson(o.root, "indexnow.json");
+    const state = stored ?? { version: 1, host, urls: {}, submissions: [] };
+    if (!stored && !o.urls?.length) {
+        notes.push("First Katman run for this host: every sitemap URL counts as new. If the site is already submitted by another script (or Cloudflare Crawler Hints), send only the pages you changed (pass urls) so IndexNow isn't flooded.");
+    }
+    if (state.host !== host) {
+        notes.push(`indexnow.json was recorded for ${state.host}; starting a fresh record for ${host}.`);
+        state.host = host;
+        state.urls = {};
+    }
+    // Candidate URLs
+    const dropped = [];
+    let entries = [];
+    let source = "sitemap";
+    if (o.urls?.length) {
+        source = "urls";
+        for (const raw of o.urls) {
+            const u = safeUrl(raw.trim(), origin);
+            if (!u || u.host !== host || !/^https?:$/.test(u.protocol))
+                dropped.push(raw);
+            else
+                entries.push({ loc: u.href, alternates: [] });
+        }
+    }
+    else {
+        const rb = await followRedirects(origin + "/robots.txt", fetcher, 3);
+        const robots = analyzeRobots(rb.final.body, origin + "/robots.txt", rb.final.status, rb.final.headers["content-type"]);
+        const sm = await loadSitemaps(origin, robots.exists ? robots.sitemaps : [], fetcher);
+        for (const e of sm.entries) {
+            const u = safeUrl(e.loc);
+            if (!u || u.host !== host)
+                dropped.push(e.loc);
+            else
+                entries.push(e);
+        }
+        if (!sm.entries.length)
+            notes.push("No sitemap URLs found: pass urls, or publish a sitemap (P4).");
+    }
+    entries = [...new Map(entries.map((e) => [e.loc, e])).values()];
+    const stats = lastmodStats(entries, now);
+    const lastmodTrusted = stats.withLastmod > 0 && !stats.identical && !stats.stampedToday;
+    if (source === "sitemap" && stats.withLastmod && !lastmodTrusted) {
+        notes.push("The sitemap's lastmod values look stamped at build time (all the same / all today), so Katman compares the pages' text instead.");
+    }
+    const candidates = [];
+    let unchanged = 0;
+    const needHash = [];
+    for (const e of entries) {
+        const rec = state.urls[e.loc];
+        if (source === "urls") {
+            candidates.push({ url: e.loc, reason: rec ? "requested" : "new" });
+            continue;
+        }
+        if (!rec) {
+            candidates.push({ url: e.loc, reason: "new", lastmod: e.lastmod });
+            // Baseline hash for next time: only worth fetching when we are about to record the submission.
+            if (!dryRun && (!lastmodTrusted || !e.lastmod))
+                needHash.push(e);
+            continue;
+        }
+        if (lastmodTrusted && e.lastmod && rec.lastmod) {
+            if (Date.parse(e.lastmod) > Date.parse(rec.lastmod))
+                candidates.push({ url: e.loc, reason: "lastmod changed", lastmod: e.lastmod });
+            else
+                unchanged++;
+            continue;
+        }
+        needHash.push(e);
+    }
+    const maxHash = o.maxHash ?? 200;
+    const toHash = needHash.slice(0, maxHash);
+    if (needHash.length > maxHash)
+        notes.push(`Only the first ${maxHash} pages were fetched to compare their text; the rest count as unchanged this time.`);
+    const hashes = await mapLimit(toHash, 4, async (e) => {
+        const r = await fetcher(e.loc, { redirect: "follow" });
+        return r.status === 200 ? textHash(r.body) : undefined;
+    });
+    const hashOf = new Map(toHash.map((e, i) => [e.loc, hashes[i]]));
+    for (const e of needHash) {
+        const rec = state.urls[e.loc];
+        const h = hashOf.get(e.loc);
+        const c = candidates.find((x) => x.url === e.loc);
+        if (c) {
+            c.hash = h;
+            continue;
+        }
+        if (rec && h && rec.hash && h !== rec.hash)
+            candidates.push({ url: e.loc, reason: "text changed", hash: h, lastmod: e.lastmod });
+        else {
+            unchanged++;
+            if (rec && h && !rec.hash && !dryRun)
+                rec.hash = h; // baseline for next time
+        }
+    }
+    const run = {
+        host, key, keyGenerated, keyLocation, keyFile, source, candidates, unchanged, dropped, lastmodTrusted, hashed: toHash.length, notes, dryRun,
+        staticDir: o.staticDir,
+    };
+    if (!dryRun && candidates.length) {
+        if (!keyOk) {
+            run.submitted = { status: -1, message: `Not submitted: ${keyLocation} ${keyFile.detail}. Publish the key file first.`, count: 0 };
+            return run;
+        }
+        let last = { status: 0, message: "", count: 0 };
+        for (let i = 0; i < candidates.length; i += 10_000) {
+            const batch = candidates.slice(i, i + 10_000);
+            const res = await post(INDEXNOW_ENDPOINT, { host, key, keyLocation, urlList: batch.map((c) => c.url) });
+            last = { status: res.status, message: responseMessage(res.status), count: batch.length };
+            state.submissions.push({ ts: now.toISOString(), count: batch.length, status: res.status, message: last.message });
+            if (res.status === 200 || res.status === 202) {
+                for (const c of batch)
+                    state.urls[c.url] = { lastmod: c.lastmod ?? state.urls[c.url]?.lastmod, hash: c.hash ?? state.urls[c.url]?.hash, submittedAt: now.toISOString() };
+            }
+            else
+                break;
+        }
+        state.submissions = state.submissions.slice(-50);
+        run.submitted = last;
+    }
+    if (!dryRun)
+        run.statePath = await writeJson(o.root, "indexnow.json", state);
+    return run;
+}
+export const MANUAL_CHECKLIST = [
+    "### Tell the search engines yourself (book ch. 13)",
+    "- [ ] **Google Search Console** (Google does not use IndexNow): add a **Domain property** verified by a DNS TXT record (covers every subdomain and protocol) → Sitemaps → submit `https://HOST/sitemap.xml` → URL Inspection → \"Request indexing\" for the important new pages (about 10 a day; if a page isn't indexed, find out why before resubmitting). Watch real AI citations in the **Generative AI performance report** (worldwide since 31 Aug 2026) — it feeds AI Overviews, AI Mode and Gemini.",
+    "- [ ] **Bing Webmaster Tools**: My Sites → Import from Search Console (or verify with a meta tag) → submit the sitemap → URL Submission for new URLs (about 100 a day). Bing's index feeds **Copilot, ChatGPT search and DuckDuckGo**. Its **AI Performance** report (Feb 2026) shows Copilot citations and grounding queries.",
+    "- [ ] **Brave Search**: no console, no sitemap submission, no IndexNow — use the re-fetch form at https://search.brave.com/submit-url for each new URL. **Claude's web search uses Brave.** Brave's crawler only fetches what robots.txt allows Googlebot; use noindex (not robots.txt) to keep a page out.",
+    "- [ ] **IndexNow** (this tool; reaches Bing, Yandex, Seznam, Naver, Yep): on every publish send only new or changed URLs — never the whole sitemap each time (the book's example site sent 173 URLs 20 times in 3 days).",
+    "- [ ] **Check the CDN once**: Cloudflare AI bot policies Search = Allow, Agent = Allow, Training ≠ Block (katman_audit_site tests this).",
+    "- [ ] **In 14 days**: Search Console → Pages. \"Discovered – currently not indexed\" usually means too few internal links; \"Crawled – currently not indexed\" means the page isn't distinct enough. Re-ask the question set (katman_visibility_run).",
+];
+export function indexNowMarkdown(r) {
+    const L = [];
+    L.push(`# IndexNow — ${r.host}${r.dryRun ? " (dry run: nothing sent)" : ""}`);
+    L.push("");
+    L.push(`Key: \`${r.key}\`${r.keyGenerated ? " (new — saved in .katman/config.json)" : ""}`);
+    if (r.keyFile.ok)
+        L.push(`Key file: ✓ ${r.keyLocation} ${r.keyFile.detail}`);
+    else {
+        L.push(`Key file: ✗ ${r.keyLocation} — ${r.keyFile.detail}. IndexNow rejects submissions (403) until this file serves exactly the key.`);
+        L.push(keyFileHelp(r.key, r.staticDir));
+    }
+    L.push("");
+    L.push(`Source: ${r.source === "urls" ? "the URLs you passed" : "the sitemap"} · ${r.candidates.length} to submit (${countBy(r.candidates.map((c) => c.reason))}) · ${r.unchanged} unchanged${r.hashed ? ` · ${r.hashed} page(s) fetched to compare text` : ""}${r.dropped.length ? ` · ${r.dropped.length} skipped (other host)` : ""}`);
+    if (r.candidates.length) {
+        L.push("");
+        L.push(mdTable(["URL", "Why"], r.candidates.slice(0, 50).map((c) => [c.url, c.reason + (c.lastmod ? ` (lastmod ${c.lastmod.slice(0, 10)})` : "")])));
+        if (r.candidates.length > 50)
+            L.push(`…and ${r.candidates.length - 50} more`);
+    }
+    for (const n of r.notes)
+        L.push(`- ${n}`);
+    if (r.dropped.length)
+        L.push(`- Skipped (IndexNow only accepts URLs on ${r.host}): ${r.dropped.slice(0, 5).join(", ")}`);
+    L.push("");
+    if (r.submitted) {
+        L.push(`**Submission:** ${r.submitted.status > 0 ? `HTTP ${r.submitted.status} — ` : ""}${r.submitted.message}${r.submitted.count ? ` (${r.submitted.count} URLs)` : ""}`);
+        if (r.statePath)
+            L.push(`Recorded in .katman/indexnow.json.`);
+    }
+    else if (r.dryRun) {
+        L.push(r.candidates.length
+            ? `Next: ${r.keyFile.ok ? "" : "publish the key file, then "}run katman_indexnow with dry_run: false to send these ${r.candidates.length} URL(s) (POST ${INDEXNOW_ENDPOINT}).`
+            : "Nothing new or changed to send.");
+    }
+    else if (!r.candidates.length) {
+        L.push("Nothing new or changed: nothing sent.");
+    }
+    L.push("");
+    L.push(MANUAL_CHECKLIST.join("\n").replace(/HOST/g, r.host));
+    return L.join("\n");
+}
+export function keyFileHelp(key, staticDir) {
+    const dir = staticDir && staticDir !== "." ? `${staticDir}/` : staticDir === "." ? "" : "public/";
+    return `Publish the key file: create \`${dir}${key}.txt\` containing exactly \`${key}\` (no newline needed), deploy, then re-run. (katman_generate can write it: artifact "indexnow-key". If a Worker/server handles every path, add a route that returns the key as text/plain.)`;
+}
+function countBy(xs) {
+    const m = new Map();
+    for (const x of xs)
+        m.set(x, (m.get(x) ?? 0) + 1);
+    return [...m.entries()].map(([k, v]) => `${v} ${k}`).join(", ") || "none";
+}