katman-mcp 0.1.0

package/dist/tools/crawl/scan/routes.js

@@ -0,0 +1,354 @@
+import { classifyPath } from "../common/routes.js";
+import { escapeRe } from "../common/text.js";
+import { codeFiles, lineOf, list, text } from "./project.js";
+const FILE_LIKE = /\.(txt|xml|json|webmanifest|ico|png|svg|jpg|jpeg|webp|rss|atom|js|css|map|pdf)$/i;
+function add(out, path, file, fileRoutes, line = 1) {
+    let p = path.trim();
+    if (!p)
+        p = "/";
+    if (!p.startsWith("/"))
+        p = "/" + p;
+    p = p.replace(/\/{2,}/g, "/");
+    if (p.length > 1)
+        p = p.replace(/\/$/, "");
+    if (/[`${}]/.test(p) && /\$\{/.test(p))
+        return; // template literal
+    const last = p.split("/").pop() ?? "";
+    if (FILE_LIKE.test(last) && !/[[:$*]/.test(last)) {
+        if (!fileRoutes.has(last.toLowerCase()))
+            fileRoutes.set(last.toLowerCase(), { file: file ?? "", line, match: p });
+        return;
+    }
+    if (out.has(p))
+        return;
+    out.set(p, { path: p, file, kind: classifyPath(p), dynamic: /[[:$*]|\(\.\.\.|\?$/.test(p) });
+}
+function segsToPath(segs) {
+    return "/" + segs.filter(Boolean).join("/");
+}
+function nextApp(rel) {
+    const m = rel.match(/(?:^|\/)app\/(.*?)\/?(?:page)\.(?:tsx|ts|jsx|js|mdx|md)$/);
+    if (!m)
+        return null;
+    const segs = m[1].split("/").filter(Boolean);
+    if (segs.some((s) => s.startsWith("_") || /^\(\.{1,3}\)/.test(s)))
+        return null;
+    return segsToPath(segs.filter((s) => !/^\(.*\)$/.test(s) && !s.startsWith("@")));
+}
+function stripIndex(segs) {
+    const out = [...segs];
+    if (out.length && /^index$/i.test(out[out.length - 1]))
+        out.pop();
+    return out;
+}
+function fileSegs(rel, base) {
+    const m = rel.match(base);
+    if (!m)
+        return null;
+    return m[1].replace(/\.[a-z0-9]+$/i, "").split("/").filter(Boolean);
+}
+export function remixFlat(name) {
+    const parts = [];
+    let cur = "";
+    let inBr = false;
+    for (const ch of name) {
+        if (ch === "[") {
+            inBr = true;
+            continue;
+        }
+        if (ch === "]") {
+            inBr = false;
+            continue;
+        }
+        if (ch === "." && !inBr) {
+            parts.push(cur);
+            cur = "";
+            continue;
+        }
+        cur += ch;
+    }
+    parts.push(cur);
+    const out = [];
+    let dynamic = false;
+    for (let s of parts) {
+        if (!s || s === "_index" || s === "route")
+            continue;
+        if (s.startsWith("_"))
+            continue;
+        if (s.endsWith("_"))
+            s = s.slice(0, -1);
+        if (s === "$") {
+            out.push("*");
+            dynamic = true;
+        }
+        else if (/^\(\$.+\)$/.test(s)) {
+            out.push(`:${s.slice(2, -1)}?`);
+            dynamic = true;
+        }
+        else if (/^\(.+\)$/.test(s))
+            out.push(`${s.slice(1, -1)}?`);
+        else if (s.startsWith("$")) {
+            out.push(`:${s.slice(1)}`);
+            dynamic = true;
+        }
+        else
+            out.push(s);
+    }
+    return { path: segsToPath(out), dynamic };
+}
+const ROUTER_LIB = /react-router|@tanstack\/react-router|@tanstack\/solid-router|vue-router|@angular\/router|@solidjs\/router|wouter|preact-router|svelte-spa-router|createBrowserRouter|RouterModule|provideRouter|<Routes\b/;
+export function scanRoutes(p, det) {
+    const out = new Map();
+    const fileRoutes = new Map();
+    const hashRouting = [];
+    const catchAll = [];
+    const pre = det.appDir ? `${det.appDir}/` : "";
+    const at = (s) => new RegExp(`^${escapeRe(pre)}${s}`);
+    const fw = det.framework.name;
+    const code = codeFiles(p);
+    if (fw === "next") {
+        for (const f of list(p, at(String.raw `(src/)?app/(.+/)?page\.(tsx|ts|jsx|js|mdx|md)$`))) {
+            const r = nextApp(f);
+            if (r !== null)
+                add(out, r, f, fileRoutes);
+        }
+        for (const f of list(p, at(String.raw `(src/)?app/(.+/)?route\.(ts|js)$`))) {
+            const m = f.match(/(?:^|\/)app\/(.*?)\/route\.(ts|js)$/);
+            if (m)
+                add(out, segsToPath(m[1].split("/").filter((s) => !/^\(.*\)$/.test(s))), f, fileRoutes);
+        }
+        for (const f of list(p, at(String.raw `(src/)?pages/.+\.(tsx|ts|jsx|js|mdx|md)$`))) {
+            const segs = fileSegs(f, /(?:^|\/)pages\/(.+)$/);
+            if (!segs || /^_(app|document|error)$/.test(segs[segs.length - 1]) || /^(404|500)$/.test(segs.join("/")))
+                continue;
+            add(out, segsToPath(stripIndex(segs)), f, fileRoutes);
+        }
+    }
+    else if (fw === "astro") {
+        for (const f of list(p, at(String.raw `src/pages/.+\.(astro|md|mdx|html|markdown|ts|js)$`))) {
+            const segs = fileSegs(f, /src\/pages\/(.+)$/);
+            if (!segs || segs.some((s) => s.startsWith("_")))
+                continue;
+            const endpoint = /\.(ts|js)$/.test(f);
+            const last = f.split("/").pop().replace(/\.(ts|js)$/, "");
+            if (endpoint && FILE_LIKE.test(last)) {
+                fileRoutes.set(last.toLowerCase(), { file: f, line: 1, match: last });
+                continue;
+            }
+            if (/^404$/.test(segs.join("/")))
+                continue;
+            add(out, segsToPath(stripIndex(segs)), f, fileRoutes);
+        }
+    }
+    else if (fw === "nuxt") {
+        for (const f of list(p, at(String.raw `(app/)?pages/.+\.vue$`))) {
+            const segs = fileSegs(f, /(?:^|\/)pages\/(.+)$/);
+            if (segs)
+                add(out, segsToPath(stripIndex(segs)), f, fileRoutes);
+        }
+        for (const f of list(p, at(String.raw `server/(api|routes)/.+\.(ts|js)$`))) {
+            const m = f.match(/server\/(api|routes)\/(.+)\.(ts|js)$/);
+            if (m)
+                add(out, (m[1] === "api" ? "/api/" : "/") + m[2].replace(/(^|\/)index$/, "").replace(/\.(get|post|put|delete)$/, ""), f, fileRoutes);
+        }
+    }
+    else if (fw === "sveltekit") {
+        for (const f of list(p, at(String.raw `src/routes/(.*/)?\+(page|server)\.(svelte|md|svx|ts|js)$`))) {
+            const m = f.match(/src\/routes\/(.*?)\/?\+(page|server)\./);
+            if (!m)
+                continue;
+            const segs = m[1].split("/").filter((s) => s && !/^\(.*\)$/.test(s));
+            add(out, segsToPath(segs), f, fileRoutes);
+        }
+    }
+    else if (fw === "remix" || fw === "react-router") {
+        const cfgRel = list(p, at(String.raw `app/routes\.(ts|js|mts)$`))[0];
+        const cfg = cfgRel ? text(p, cfgRel) : "";
+        if (cfg && !/flatRoutes\s*\(/.test(cfg)) {
+            for (const m of cfg.matchAll(/\broute\(\s*["'`]([^"'`]*)["'`]\s*,\s*["'`]([^"'`]+)["'`]/g))
+                add(out, m[1], `${pre}app/${m[2]}`, fileRoutes);
+            for (const m of cfg.matchAll(/\bindex\(\s*["'`]([^"'`]+)["'`]/g))
+                add(out, "/", `${pre}app/${m[1]}`, fileRoutes);
+        }
+        for (const f of list(p, at(String.raw `app/routes/[^/]+\.(tsx|ts|jsx|js|mdx|md)$`))) {
+            const name = f.split("/").pop().replace(/\.(tsx|ts|jsx|js|mdx|md)$/, "");
+            const r = remixFlat(name);
+            if (!/export\s+default/.test(text(p, f)) && /\.(ts|js)$/.test(f)) {
+                const last = r.path.split("/").pop() ?? "";
+                if (FILE_LIKE.test(last))
+                    fileRoutes.set(last.toLowerCase(), { file: f, line: 1, match: r.path });
+                else
+                    add(out, r.path, f, fileRoutes);
+                continue;
+            }
+            add(out, r.path, f, fileRoutes);
+        }
+        for (const f of list(p, at(String.raw `app/routes/[^/]+/route\.(tsx|ts|jsx|js)$`))) {
+            const name = f.split("/").slice(-2, -1)[0];
+            add(out, remixFlat(name).path, f, fileRoutes);
+        }
+    }
+    else if (fw === "gatsby") {
+        for (const f of list(p, at(String.raw `src/pages/.+\.(tsx|ts|jsx|js)$`))) {
+            const segs = fileSegs(f, /src\/pages\/(.+)$/);
+            if (!segs || /^404$/.test(segs.join("/")))
+                continue;
+            add(out, segsToPath(stripIndex(segs)), f, fileRoutes);
+        }
+    }
+    else if (fw === "hugo") {
+        for (const f of list(p, /^content\/.+\.(md|markdown|html)$/)) {
+            const segs = fileSegs(f, /^content\/(.+)$/);
+            const last = segs[segs.length - 1];
+            if (last === "_index" || last === "index")
+                segs.pop();
+            add(out, segsToPath(segs), f, fileRoutes);
+        }
+    }
+    else if (fw === "jekyll") {
+        for (const f of list(p, /^[^_/][^/]*\.(md|markdown|html)$/)) {
+            if (/^(README|CHANGELOG|LICENSE|404)\./i.test(f))
+                continue;
+            add(out, segsToPath(stripIndex([f.replace(/\.(md|markdown|html)$/, "")])), f, fileRoutes);
+        }
+        for (const f of list(p, /^_posts\/.+\.(md|markdown|html)$/)) {
+            const slug = f.split("/").pop().replace(/^\d{4}-\d{2}-\d{2}-/, "").replace(/\.(md|markdown|html)$/, "");
+            add(out, `/${slug}`, f, fileRoutes);
+        }
+    }
+    else if (fw === "eleventy" || fw === "static-html" || fw === "vitepress" || fw === "docusaurus") {
+        const input = fw === "eleventy" && p.dirs.has("src") ? "src/" : fw === "vitepress" ? "docs/" : fw === "docusaurus" ? "" : pre;
+        const re = fw === "static-html" ? /\.html?$/ : /\.(md|njk|liquid|html|11ty\.js|mdx)$/;
+        for (const f of p.files.keys()) {
+            if (!f.startsWith(input) || !re.test(f))
+                continue;
+            if (/(^|\/)(_includes|_data|_layouts|node_modules|\.vitepress)\//.test(f) || /(^|\/)(README|CHANGELOG|LICENSE)\.md$/i.test(f))
+                continue;
+            if (fw === "docusaurus" && !/^(docs|blog|src\/pages)\//.test(f))
+                continue;
+            const segs = f.slice(input.length).replace(/\.(html?|md|njk|liquid|11ty\.js|mdx)$/, "").split("/");
+            if (/^404$/.test(segs.join("/")))
+                continue;
+            add(out, segsToPath(stripIndex(segs)), f, fileRoutes);
+        }
+    }
+    else if (fw === "tanstack-start") {
+        for (const f of code) {
+            for (const m of f.text.matchAll(/createFileRoute\(\s*["'`]([^"'`]+)["'`]\s*\)/g))
+                add(out, tanstackPath(m[1]), f.rel, fileRoutes, lineOf(f.text, m.index ?? 0));
+        }
+    }
+    // SPA routers (also used inside other frameworks' client code)
+    if (!out.size || ["vite-react", "vite-vue", "vite-svelte", "vite-solid", "vite", "cra", "angular", "static-html", "unknown", "tanstack-start"].includes(fw)) {
+        spaRoutes(code, out, fileRoutes, catchAll, fw);
+    }
+    // Hand-written servers (Workers / Express / Hono) and path tables
+    if (fw === "custom-server" || fw === "unknown" || !out.size)
+        serverRoutes(code, out, fileRoutes);
+    // Prebuilt HTML served from the static folder by a custom server (e.g. public/pricing.html → /pricing)
+    if (fw === "custom-server" && det.staticDir) {
+        const pretty = det.hosting.provider === "cloudflare" || det.hosting.provider === "vercel" || det.hosting.provider === "netlify";
+        for (const f of p.files.keys()) {
+            if (!f.startsWith(det.staticDir + "/") || !/\.html?$/.test(f))
+                continue;
+            const rel = f.slice(det.staticDir.length + 1);
+            if (/(^|\/)(404|500)\.html?$/.test(rel))
+                continue;
+            const segs = rel.split("/");
+            const last = segs.pop();
+            if (!/^index\.html?$/.test(last))
+                segs.push(pretty ? last.replace(/\.html?$/, "") : last);
+            add(out, segsToPath(segs), f, fileRoutes);
+        }
+    }
+    // Machine files served from code in any framework (robots.txt / sitemap.xml / llms.txt routes)
+    collectFileRoutes(code, fileRoutes);
+    // Hash routing and catch-all routes
+    for (const f of code) {
+        const t = f.text;
+        for (const m of t.matchAll(/\b(HashRouter|createHashRouter|createWebHashHistory|useHash\s*:\s*true|withHashLocation|hashHistory|createHashHistory)\b/g)) {
+            hashRouting.push({ file: f.rel, line: lineOf(t, m.index ?? 0), match: m[1] });
+        }
+        if (/from\s+["']svelte-spa-router["']/.test(t))
+            hashRouting.push({ file: f.rel, line: 1, match: "svelte-spa-router (hash-based)" });
+    }
+    return { routes: [...out.values()], hashRouting, catchAll, fileRoutes };
+}
+function tanstackPath(p) {
+    const segs = p.split("/").filter((s) => s && !s.startsWith("_") && !/^\(.*\)$/.test(s));
+    return segsToPath(segs);
+}
+function spaRoutes(code, out, fileRoutes, catchAll, fw) {
+    for (const f of code) {
+        const t = f.text;
+        const isRouterFile = ROUTER_LIB.test(t);
+        for (const m of t.matchAll(/<Route\b[^>]*?\bpath\s*=\s*(?:\{\s*)?["'`]([^"'`]*)["'`]/g)) {
+            const line = lineOf(t, m.index ?? 0);
+            if (m[1] === "*" || m[1] === "/*")
+                catchAll.push({ file: f.rel, line, match: `path="${m[1]}"` });
+            else
+                add(out, m[1], f.rel, fileRoutes, line);
+        }
+        if (isRouterFile) {
+            for (const m of t.matchAll(/\bpath\s*:\s*["'`]([^"'`]*)["'`]/g)) {
+                const line = lineOf(t, m.index ?? 0);
+                const v = m[1];
+                if (v === "*" || v === "**" || v === "/*" || /pathMatch\(\.\*\)/.test(v) || v === "/:catchAll(.*)")
+                    catchAll.push({ file: f.rel, line, match: `path: "${v}"` });
+                else if (fw === "angular" || !v.startsWith("/"))
+                    add(out, v, f.rel, fileRoutes, line);
+                else
+                    add(out, v, f.rel, fileRoutes, line);
+            }
+            for (const m of t.matchAll(/createFileRoute\(\s*["'`]([^"'`]+)["'`]\s*\)/g))
+                add(out, tanstackPath(m[1]), f.rel, fileRoutes, lineOf(t, m.index ?? 0));
+        }
+        if (/from\s+["']svelte-spa-router["']/.test(t)) {
+            for (const m of t.matchAll(/["'`](\/[^"'`]*)["'`]\s*:\s*[A-Za-z_]\w*/g)) {
+                if (m[1] === "*")
+                    catchAll.push({ file: f.rel, line: lineOf(t, m.index ?? 0), match: "*" });
+                else
+                    add(out, m[1], f.rel, fileRoutes, lineOf(t, m.index ?? 0));
+            }
+        }
+    }
+}
+const SERVER_CALL = /\b(?:app|router|r|route|routes|server|api|hono|fastify|site|web)\.(?:get|all|on|route)\(\s*(?:["'`](?:GET|HEAD)["'`]\s*,\s*)?["'`](\/[^"'`\s]*)["'`]/g;
+const PATHNAME_EQ = /\b(?:pathname|url\.pathname)\s*===?\s*["'`](\/[^"'`\s]*)["'`]/g;
+const CASE_PATH = /case\s+["'`](\/[^"'`\s]*)["'`]\s*:/g;
+const PATH_TABLE_FILE = /(^|\/)(paths?|routes?|urls?|pages|site-?map|nav|navigation|links|menu)\.(m?[jt]sx?|cjs)$/i;
+const NOT_ROUTES_FILE = /(^|\/)(__mocks__|mocks?|fakes?|stubs?|fixtures?|seeds?|samples?)(\/|[._-])|(^|\/)[^/]*(mock|fake|stub|dev[-_.]?server)[^/]*\.[a-z]+$/i;
+function serverRoutes(code, out, fileRoutes) {
+    for (const f of code) {
+        if (!/\.(m?[jt]sx?|cjs)$/.test(f.rel) || NOT_ROUTES_FILE.test(f.rel))
+            continue;
+        const t = f.text;
+        for (const re of [SERVER_CALL, PATHNAME_EQ, CASE_PATH]) {
+            for (const m of t.matchAll(re))
+                add(out, m[1], f.rel, fileRoutes, lineOf(t, m.index ?? 0));
+        }
+        if (PATH_TABLE_FILE.test(f.rel)) {
+            for (const m of t.matchAll(/["'`](\/|\/[a-z0-9][a-z0-9\-/]*)["'`]/g)) {
+                if (/\/(static|assets|img|images|fonts|css|js)\//.test(m[1]))
+                    continue;
+                add(out, m[1], f.rel, fileRoutes, lineOf(t, m.index ?? 0));
+            }
+        }
+    }
+}
+function collectFileRoutes(code, fileRoutes) {
+    const names = ["robots.txt", "sitemap.xml", "sitemap-index.xml", "sitemap_index.xml", "llms.txt", "llms-full.txt"];
+    for (const f of code) {
+        if (!/\.(m?[jt]sx?|cjs)$/.test(f.rel))
+            continue;
+        const t = f.text;
+        for (const n of names) {
+            if (fileRoutes.has(n))
+                continue;
+            const re = new RegExp(`(?:\\.(?:get|all)\\(|pathname\\s*===?\\s*|case\\s+|path\\s*:\\s*)["'\`]/${escapeRe(n)}["'\`]`);
+            const m = t.match(re);
+            if (m)
+                fileRoutes.set(n, { file: f.rel, line: lineOf(t, m.index ?? 0), match: `/${n}` });
+        }
+    }
+}