katman-mcp 0.1.0

package/dist/tools/crawl/scan/index.js

@@ -0,0 +1,175 @@
+import { mdTable, sortFindings, truncate } from "../../../util.js";
+import { CSR_ACCEPT, JS_RENDER_NOTE, csrFixFor } from "../common/advice.js";
+import { findingsList, severityLine } from "../common/findings.js";
+import { detect } from "./framework.js";
+import { loadProject } from "./project.js";
+import { scanRoutes } from "./routes.js";
+import { detectAnalytics, detectFiles, detectHeadSource, detectI18n, detectIndexingPitfalls, detectLinksAndImages, jsonLdTypes } from "./signals.js";
+const KIND_ORDER = { public: 0, legal: 1, search: 2, checkout: 3, auth: 4, app: 5, api: 6, unknown: 7 };
+export async function runScan(root, opts = {}) {
+    const project = await loadProject(root, { maxFiles: opts.maxFiles });
+    const det = detect(project);
+    const rs = scanRoutes(project, det);
+    const files = detectFiles(project, det, rs);
+    const head = detectHeadSource(project, det, rs);
+    const i18n = detectI18n(project, det);
+    const analytics = detectAnalytics(project, det.pkg?.deps);
+    const findings = [
+        ...det.findings,
+        ...files.findings,
+        ...head.findings,
+        ...i18n.findings,
+        ...analytics.findings,
+        ...detectLinksAndImages(project),
+        ...detectIndexingPitfalls(project),
+        ...routeFindings(det, rs, files.files),
+    ];
+    const routes = [...rs.routes]
+        .sort((a, b) => (KIND_ORDER[a.kind] ?? 9) - (KIND_ORDER[b.kind] ?? 9) || Number(a.dynamic) - Number(b.dynamic) || a.path.localeCompare(b.path))
+        .slice(0, 2000);
+    const notes = [...head.notes];
+    if (project.truncated)
+        notes.push(`File cap reached (${project.stats.listed} files): pass a larger max_files or scan the app folder.`);
+    const result = {
+        version: 1,
+        ts: (opts.now ?? new Date()).toISOString(),
+        root,
+        framework: det.framework,
+        rendering: det.rendering,
+        hosting: det.hosting,
+        ...(det.domains.length ? { domains: det.domains } : {}),
+        staticDir: det.staticDir,
+        routes,
+        files: { ...files.files, headSource: head.info },
+        jsonLdTypes: jsonLdTypes(project),
+        i18n: i18n.i18n,
+        analytics: analytics.analytics,
+        findings: dedupe(sortFindings(findings)),
+    };
+    return { result, project, det, routeScan: rs, notes };
+}
+function dedupe(fs) {
+    const seen = new Set();
+    return fs.filter((f) => (seen.has(f.id) ? false : (seen.add(f.id), true)));
+}
+function routeFindings(det, rs, files) {
+    const out = [];
+    const csr = det.rendering.verdict === "csr";
+    if (rs.hashRouting.length) {
+        out.push({
+            id: "scan-hash-routing",
+            title: "Hash routing: pages live behind # (e.g. /#/pricing)",
+            severity: "critical",
+            gate: "read",
+            prompt: "P2",
+            detail: `${rs.hashRouting.map((h) => `${h.file}:${h.line} ${h.match}`).slice(0, 3).join("; ")}. Everything after # never reaches the server, so crawlers see one URL for the whole site. ${JS_RENDER_NOTE}`,
+            files: rs.hashRouting.map((h) => `${h.file}:${h.line}`),
+            fix: `Switch to path routing (BrowserRouter / createBrowserRouter / createWebHistory) and serve each public path with its own HTML. ${csrFixFor(det.framework.name, det.framework.router)}`,
+            accept: `Every public page has its own path (no #/…) and returns its own HTML; ${CSR_ACCEPT}`,
+        });
+    }
+    const fallback = det.spa.spaFallback;
+    if (csr || fallback) {
+        const ev = [...rs.catchAll.map((c) => `${c.file}:${c.line} ${c.match}`), ...(fallback ? [fallback] : [])];
+        out.push({
+            id: "scan-spa-soft-404",
+            title: "Unknown URLs will answer 200 (single-page-app fallback → soft 404)",
+            severity: "high",
+            gate: "read",
+            prompt: "P11",
+            detail: `The server returns index.html for every path and the app shows "not found" in the browser, with status 200. Google counts that as a soft 404 (book trap #3).${ev.length ? ` Evidence: ${ev.slice(0, 3).join("; ")}.` : ""}`,
+            files: [...rs.catchAll.map((c) => c.file)].slice(0, 5),
+            fix: "Return a real 404 for unknown paths: list the valid routes at the edge/server (or prerender them and add a 404.html; Cloudflare Workers assets: not_found_handling \"404-page\"), or at least put noindex on the not-found screen.",
+            accept: 'curl -s -o /dev/null -w "%{http_code}" https://<domain>/does-not-exist prints 404',
+        });
+    }
+    if (!files.robots.exists) {
+        out.push({ id: "scan-no-robots", title: "No robots.txt in the codebase", severity: "medium", gate: "read", prompt: "P4",
+            detail: "Neither a static robots.txt nor a generator was found. Not required, but the book's file names each AI bot, closes only app paths and points to the sitemap.",
+            fix: "Add robots.txt to the static folder (book appendix A; katman_generate can write it from research).", accept: "robots.txt has no `Disallow: /` for *, names the AI bots and has a Sitemap line" });
+    }
+    if (!files.sitemap.exists) {
+        out.push({ id: "scan-no-sitemap", title: "No sitemap in the codebase", severity: "high", gate: "read", prompt: "P4",
+            detail: "No static sitemap.xml, generator route, plugin or build script was found.",
+            fix: "Generate sitemap.xml at build time from the page list: only 200, canonical, indexable URLs with a real lastmod.", accept: "sitemap.xml lists every public page; every URL in it returns 200 and is canonical" });
+    }
+    if (!files.llms.exists) {
+        out.push({ id: "scan-no-llms", title: "No llms.txt", severity: "info", gate: "quote", prompt: "P8",
+            detail: "Optional: Google ignores it and most files get no AI-bot requests. If you add one, keep it short and generated from the same data as the pages.",
+            fix: "katman_generate can write llms.txt from research (book appendix B).", accept: "llms.txt and the pages do not contradict each other on any price or claim" });
+    }
+    if (!files.notFound.exists && !csr && det.framework.name !== "unknown") {
+        out.push({ id: "scan-no-404-page", title: "No custom 404 page", severity: "low", gate: "read", prompt: "P11",
+            detail: "The platform default still returns 404, but a custom page can link visitors (and crawlers) back to key pages.",
+            fix: "Add a 404 page (with noindex) linking to the homepage and main sections.", accept: "Unknown URLs return 404 with a helpful page" });
+    }
+    const bad = rs.routes.filter((r) => r.kind === "public" && !r.dynamic && (/[A-Z_]/.test(r.path) || r.path.length > 100));
+    if (bad.length) {
+        out.push({ id: "scan-route-hygiene", title: `${bad.length} public route${bad.length === 1 ? " uses" : "s use"} uppercase/underscores or very long paths`, severity: "low", gate: "read", prompt: "P11",
+            detail: bad.slice(0, 5).map((r) => r.path).join(", "), fix: "Use lowercase, hyphenated paths for new pages; 301 old ones if you rename.", accept: "Public paths are lowercase and hyphenated" });
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Markdown
+const Y = "✓";
+const N = "✗";
+function fileRow(label, info, missingHint) {
+    return [label, info.exists ? Y : N, info.exists ? `${info.path ?? ""}${info.generated === true ? " (generated)" : info.generated === false ? " (static file)" : ""}` : missingHint];
+}
+export function scanMarkdown(run, opts = {}) {
+    const r = run.result;
+    const L = [];
+    L.push(`# Katman code scan — ${r.root}`);
+    if (opts.warning)
+        L.push(`> ${opts.warning}`);
+    const fw = r.framework;
+    L.push(`Framework: **${fw.name}${fw.version ? ` ${fw.version}` : ""}**${fw.router ? ` (${fw.router})` : ""}${fw.builder ? ` · builder: ${fw.builder}` : ""}${r.hosting.provider ? ` · hosting: ${r.hosting.provider}` : ""}${run.det.appDir ? ` · app folder: ${run.det.appDir}/` : ""}`);
+    L.push(`Rendering: **${r.rendering.verdict.toUpperCase()}** (${r.rendering.confidence} confidence)`);
+    for (const e of r.rendering.evidence.slice(0, 5))
+        L.push(`- ${e}`);
+    if (r.hosting.previewPatterns.length)
+        L.push(`Preview/default addresses to check: ${r.hosting.previewPatterns.join(", ")}`);
+    if (run.det.domains.length)
+        L.push(`Live site in config: ${run.det.domains.map((d) => `https://${d}`).join(", ")} → audit it with katman_audit_site.`);
+    if (r.staticDir)
+        L.push(`Static folder (robots.txt, llms.txt, IndexNow key file go here): \`${r.staticDir}/\``);
+    L.push("");
+    L.push("## Files");
+    const sd = r.staticDir ? `${r.staticDir}/` : "";
+    const rows = [
+        fileRow("robots.txt", r.files.robots, `missing → ${sd}robots.txt (P4)`),
+        fileRow("sitemap.xml", r.files.sitemap, "missing → generate at build time (P4)"),
+        fileRow("llms.txt", r.files.llms, `optional → ${sd}llms.txt (P8)`),
+        fileRow("llms-full.txt", r.files.llmsFull, "optional"),
+        fileRow("404 page", r.files.notFound, "missing (P11)"),
+        fileRow("Head tags source", r.files.headSource ?? { exists: false }, "none found (P3)"),
+        ["JSON-LD", r.jsonLdTypes.length ? Y : N, r.jsonLdTypes.length ? truncate(r.jsonLdTypes.join(", "), 120) : "none (P5)"],
+        ["i18n", r.i18n.detected ? Y : "–", `${r.i18n.locales.length ? r.i18n.locales.join(", ") : "single language"}${r.i18n.library ? ` (${r.i18n.library})` : ""} · <html lang>: ${r.i18n.htmlLang ?? "not found"}`],
+        ["Analytics", r.analytics.providers.length ? Y : N, `${r.analytics.providers.join(", ") || "none"} · AI channel (chatgpt.com / utm_source) tracked: ${r.analytics.tracksAiChannel ? Y : N}`],
+    ];
+    L.push(mdTable(["File / signal", "Found", "Where"], rows));
+    L.push("");
+    const kinds = {};
+    for (const x of r.routes)
+        kinds[x.kind] = (kinds[x.kind] ?? 0) + 1;
+    L.push(`## Routes — ${r.routes.length} (${Object.entries(kinds).map(([k, v]) => `${v} ${k}`).join(" · ") || "none found"})`);
+    if (r.routes.length) {
+        L.push(mdTable(["Path", "Kind", "Dynamic", "File"], r.routes.slice(0, 60).map((x) => [truncate(x.path, 60), x.kind, x.dynamic ? "yes" : "", truncate(x.file ?? "", 60)])));
+        if (r.routes.length > 60)
+            L.push(`…${r.routes.length - 60} more in .katman/scan.json`);
+    }
+    L.push("");
+    L.push(`## Findings — ${severityLine(r.findings)}`);
+    L.push(findingsList(r.findings, { limit: 40 }));
+    if (r.findings.length > 40)
+        L.push(`…${r.findings.length - 40} more in .katman/scan.json`);
+    L.push("");
+    const s = run.project.stats;
+    L.push(`Scanned ${s.listed} files (read ${s.read}, ${Math.round(s.bytesRead / 1024)} KB); skipped ${s.skippedSecret} secret-looking file(s) unopened, ${s.skippedSymlink} symlink(s), ${s.skippedLarge} large file(s).${opts.savedTo ? ` Saved: ${opts.savedTo}` : ""}`);
+    for (const n of run.notes)
+        L.push(`- ${n}`);
+    L.push("");
+    L.push("Next: katman_audit_site checks what the live site actually serves (bots, JS-off text, 404s, sitemap); then fix the findings in order with katman_prompt.");
+    return L.join("\n");
+}

package/dist/tools/crawl/scan/project.js

@@ -0,0 +1,180 @@
+// Safe, bounded project walker for katman_scan_code.
+//  - never follows symlinks
+//  - skips dependency/build/cache folders
+//  - NEVER opens secret-looking files (.env*, .dev.vars*, *.pem, *.key, *.p12, *secret*, *credential*, service
+//    account JSON, key files…) — not even to report that they exist
+//  - caps the number of files (default 5000), the size of each file (512 KB) and the total bytes read
+import { promises as fs } from "node:fs";
+import path from "node:path";
+export const SKIP_DIRS = new Set([
+    "node_modules", ".git", "dist", "build", "out", ".next", ".nuxt", ".svelte-kit", ".output", ".vercel", ".netlify",
+    ".wrangler", "coverage", ".katman", ".turbo", ".cache",
+    // other generated / dependency / tooling folders
+    ".parcel-cache", ".astro", ".docusaurus", ".expo", ".yarn", ".pnpm-store", "bower_components", "vendor", ".venv",
+    "venv", "__pycache__", ".gradle", "Pods", ".angular", ".vite", "storybook-static", ".firebase", ".serverless",
+    ".terraform", ".idea", ".vscode", ".husky", ".changeset", "target", "tmp", ".tmp", "logs", "backups",
+]);
+/** Hidden folders worth walking (builder signatures, workflows); other dot-folders are only noted by name. */
+const WALK_HIDDEN = new Set([".github", ".bolt", ".cursor", ".idx", ".well-known", ".lovable", ".storybook", ".claude"]);
+export function isNeverRead(name) {
+    const n = name.toLowerCase();
+    if (n.startsWith(".env") || n.startsWith(".dev.vars"))
+        return true;
+    if (/\.(pem|key|p12|pfx|jks|keystore|ppk|kdbx|gpg|asc|der|crt|cer|csr)$/.test(n))
+        return true;
+    if (n.includes("secret") || n.includes("credential"))
+        return true;
+    if (/(^|[._-])(api[-_]?key|access[-_]?token|auth[-_]?token|private[-_]?key)([._-]|$)/.test(n))
+        return true;
+    if (/[-_]key$/.test(n) || n === "key" || n.startsWith(".key"))
+        return true;
+    if (/service[-_]?account|adminsdk|keyfile|(^|[-_.])keys?\.json$/.test(n))
+        return true;
+    if ([".npmrc", ".netrc", ".pypirc", ".htpasswd", ".git-credentials", "id_rsa", "id_ed25519", "id_ecdsa", ".replit"].includes(n))
+        return true;
+    if (/^id_(rsa|ed25519|ecdsa|dsa)/.test(n))
+        return true;
+    return false;
+}
+const TEXT_EXT = new Set([
+    ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".vue", ".svelte", ".astro", ".html", ".htm", ".md",
+    ".mdx", ".markdown", ".json", ".jsonc", ".json5", ".toml", ".yaml", ".yml", ".txt", ".xml", ".liquid", ".njk",
+    ".hbs", ".handlebars", ".ejs", ".pug", ".erb", ".php", ".twig", ".webmanifest", ".svx", ".rb", ".py",
+]);
+const TEXT_NAMES = new Set(["_headers", "_redirects", "procfile", "gemfile", "cname", "dockerfile", ".htaccess", "robots.txt"]);
+const NO_READ = new Set(["package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb", "npm-shrinkwrap.json", "composer.lock", "gemfile.lock", "cargo.lock", "poetry.lock"]);
+export async function loadProject(root, opts = {}) {
+    const maxFiles = opts.maxFiles ?? 5000;
+    const maxFileBytes = opts.maxFileBytes ?? 512 * 1024;
+    const maxTotalBytes = opts.maxTotalBytes ?? 96 * 1024 * 1024;
+    const p = {
+        root,
+        files: new Map(),
+        dirs: new Set([""]),
+        seenDirNames: new Set(),
+        truncated: false,
+        stats: { listed: 0, read: 0, bytesRead: 0, skippedSecret: 0, skippedSymlink: 0, skippedLarge: 0 },
+    };
+    const queue = [""];
+    while (queue.length) {
+        const rel = queue.shift();
+        let entries;
+        try {
+            entries = await fs.readdir(path.join(root, rel), { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        entries.sort((a, b) => a.name.localeCompare(b.name));
+        for (const e of entries) {
+            const childRel = rel ? `${rel}/${e.name}` : e.name;
+            if (e.isSymbolicLink()) {
+                p.stats.skippedSymlink++;
+                continue;
+            }
+            if (e.isDirectory()) {
+                if (isNeverRead(e.name)) {
+                    p.stats.skippedSecret++;
+                    continue;
+                }
+                if (SKIP_DIRS.has(e.name) || (e.name.startsWith(".") && !WALK_HIDDEN.has(e.name))) {
+                    p.seenDirNames.add(e.name);
+                    continue;
+                }
+                p.dirs.add(childRel);
+                queue.push(childRel);
+                continue;
+            }
+            if (!e.isFile())
+                continue;
+            if (isNeverRead(e.name)) {
+                p.stats.skippedSecret++;
+                continue;
+            }
+            if (p.stats.listed >= maxFiles) {
+                p.truncated = true;
+                queue.length = 0;
+                break;
+            }
+            let size = 0;
+            try {
+                size = (await fs.lstat(path.join(root, childRel))).size;
+            }
+            catch {
+                continue;
+            }
+            p.stats.listed++;
+            const file = { rel: childRel, size };
+            p.files.set(childRel, file);
+            const lower = e.name.toLowerCase();
+            const ext = path.extname(lower);
+            if (NO_READ.has(lower) || !(TEXT_EXT.has(ext) || TEXT_NAMES.has(lower)))
+                continue;
+            if (size > maxFileBytes) {
+                p.stats.skippedLarge++;
+                continue;
+            }
+            if (p.stats.bytesRead + size > maxTotalBytes)
+                continue;
+            try {
+                const text = await fs.readFile(path.join(root, childRel), "utf8");
+                p.stats.read++;
+                p.stats.bytesRead += size;
+                file.text = text;
+                file.minified = /\.min\.(js|css)$/.test(lower) || (text.length > 20_000 && text.split("\n").length < text.length / 1500);
+            }
+            catch {
+                /* unreadable: keep as listed */
+            }
+        }
+    }
+    return p;
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Query helpers
+export function has(p, rel) {
+    return p.files.has(rel);
+}
+export function text(p, rel) {
+    return p.files.get(rel)?.text ?? "";
+}
+export function hasDir(p, rel) {
+    return p.dirs.has(rel);
+}
+export function list(p, re) {
+    const out = [];
+    for (const k of p.files.keys())
+        if (re.test(k))
+            out.push(k);
+    return out;
+}
+export function first(p, rels) {
+    return rels.find((r) => p.files.has(r));
+}
+const CODE_EXT = /\.(jsx?|tsx?|mjs|cjs|mts|cts|vue|svelte|astro|html?|njk|liquid|hbs|ejs|mdx|svx)$/i;
+const TEST_PATH = /(^|\/)(__tests__|__mocks__|tests?|e2e|cypress|playwright|spec|fixtures|stories|storybook-static)(\/|$)|\.(test|spec|stories)\.[a-z]+$/i;
+/** Source files worth pattern-scanning: code, not tests, not minified bundles. */
+export function codeFiles(p, opts = {}) {
+    const out = [];
+    for (const f of p.files.values()) {
+        if (!f.text || f.minified)
+            continue;
+        if (!CODE_EXT.test(f.rel))
+            continue;
+        if (TEST_PATH.test(f.rel))
+            continue;
+        if (!opts.includeScripts && /(^|\/)scripts\//.test(f.rel))
+            continue;
+        if (/(^|\/)(public|static)\/.*\.js$/.test(f.rel))
+            continue; // shipped bundles / vendored browser scripts
+        out.push(f);
+    }
+    return out;
+}
+export function lineOf(textIn, index) {
+    let n = 1;
+    for (let i = 0; i < index && i < textIn.length; i++)
+        if (textIn.charCodeAt(i) === 10)
+            n++;
+    return n;
+}