katman-mcp 0.1.0

package/dist/tools/crawl/scan/signals.js

@@ -0,0 +1,414 @@
+import { crawlerByToken } from "../../../crawlers.js";
+import { classifyPath } from "../common/routes.js";
+import { analyzeRobots } from "../audit/robots.js";
+import { lastmodStats, parseSitemapXml } from "../audit/sitemap.js";
+import { codeFiles, first, has, lineOf, list, text } from "./project.js";
+function F(id, severity, gate, prompt, title, detail, extra = {}) {
+    return { id, title, severity, gate, prompt, detail, ...extra };
+}
+function exportedFn(code, names) {
+    const re = new RegExp(`export\\s+(?:async\\s+)?(?:function\\s+(?:${names.join("|")})\\b|const\\s+(?:${names.join("|")})\\s*=)`);
+    return code.find((f) => /\.(m?[jt]sx?|cjs)$/.test(f.rel) && re.test(f.text))?.rel;
+}
+export function detectFiles(p, det, rs) {
+    const pre = det.appDir ? `${det.appDir}/` : "";
+    const sd = det.staticDir && det.staticDir !== "." ? `${det.staticDir}/` : det.staticDir === "." ? "" : `${pre}public/`;
+    const deps = det.pkg?.deps ?? {};
+    const code = codeFiles(p, { includeScripts: true });
+    const findings = [];
+    const staticFile = (name) => first(p, [`${sd}${name}`, `${pre}public/${name}`, `${pre}static/${name}`, `${pre}${name}`, name]);
+    const gen = (res) => {
+        for (const re of res) {
+            const hit = list(p, re)[0];
+            if (hit)
+                return hit;
+        }
+        return undefined;
+    };
+    const fromRoute = (name, fnNames) => {
+        const hit = rs.fileRoutes.get(name);
+        if (!hit)
+            return undefined;
+        return exportedFn(code, fnNames) ?? hit.file;
+    };
+    // robots.txt
+    const staticRobots = staticFile("robots.txt");
+    const robotsGen = gen([/(^|\/)(src\/)?app\/robots\.(ts|js|tsx)$/, /(^|\/)src\/pages\/robots\.txt\.(ts|js)$/, /(^|\/)src\/routes\/robots\.txt\/\+server\.(ts|js)$/, /(^|\/)server\/routes\/robots\.txt\.(ts|js)$/, /(^|\/)app\/routes\/(robots\[\.\]txt|\[robots\.txt\])\.(tsx?|jsx?)$/, /(^|\/)pages\/robots\.txt\.(tsx?|jsx?)$/, /(^|\/)app\/robots\.txt\/route\.(ts|js)$/]) ??
+        fromRoute("robots.txt", ["robots", "robotsTxt", "getRobots"]) ??
+        (["next-sitemap", "@nuxtjs/robots", "nuxt-simple-robots", "astro-robots-txt", "gatsby-plugin-robots-txt"].find((d) => d in deps) ? `dependency: ${["next-sitemap", "@nuxtjs/robots", "nuxt-simple-robots", "astro-robots-txt", "gatsby-plugin-robots-txt"].find((d) => d in deps)}` : undefined) ??
+        code.find((f) => /\.(m?[jt]s|cjs)$/.test(f.rel) && /User-agent:\s*[*\w]/.test(f.text) && /Sitemap:/.test(f.text) && !/(^|\/)test/.test(f.rel))?.rel;
+    const robots = staticRobots ? { exists: true, path: staticRobots, generated: false } : robotsGen ? { exists: true, path: robotsGen, generated: true } : { exists: false };
+    // sitemap
+    const staticSitemap = staticFile("sitemap.xml") ?? staticFile("sitemap-index.xml") ?? staticFile("sitemap_index.xml");
+    const sitemapDep = ["next-sitemap", "@astrojs/sitemap", "@nuxtjs/sitemap", "nuxt-simple-sitemap", "gatsby-plugin-sitemap", "vite-plugin-sitemap", "svelte-sitemap", "super-sitemap", "sitemap", "react-router-sitemap", "@11ty/eleventy-plugin-sitemap"].find((d) => d in deps);
+    const sitemapGen = gen([/(^|\/)(src\/)?app\/sitemap\.(ts|js|tsx)$/, /(^|\/)app\/sitemap\.xml\/route\.(ts|js)$/, /(^|\/)pages\/sitemap\.xml\.(tsx?|jsx?)$/, /(^|\/)src\/pages\/sitemap[^/]*\.xml\.(ts|js)$/, /(^|\/)src\/routes\/sitemap\.xml\/\+server\.(ts|js)$/, /(^|\/)server\/routes\/sitemap\.xml\.(ts|js)$/, /(^|\/)app\/routes\/(sitemap\[\.\]xml|\[sitemap\.xml\])\.(tsx?|jsx?)$/, /(^|\/)(sitemap\.xml|sitemap)\.(njk|liquid|11ty\.js)$/]) ??
+        fromRoute("sitemap.xml", ["sitemap", "sitemapXml", "getSitemap"]) ??
+        (sitemapDep ? `dependency: ${sitemapDep}` : undefined) ??
+        list(p, /(^|\/)scripts\/[^/]*sitemap[^/]*\.(m?[jt]s|cjs|py)$/i)[0] ??
+        code.find((f) => /\.(m?[jt]s|cjs)$/.test(f.rel) && /<urlset\b/.test(f.text))?.rel ??
+        (det.framework.name === "hugo" ? "hugo (built-in sitemap)" : undefined) ??
+        (/jekyll-sitemap/.test(text(p, "_config.yml") + text(p, "Gemfile")) ? "jekyll-sitemap plugin" : undefined);
+    const sitemap = staticSitemap ? { exists: true, path: staticSitemap, generated: false } : sitemapGen ? { exists: true, path: sitemapGen, generated: true } : { exists: false };
+    const sitemapGenerators = [sitemapGen, ...list(p, /(^|\/)next-sitemap\.config\.[mc]?js$/)].filter((x) => !!x && !x.startsWith("dependency") && has(p, x));
+    // llms.txt / llms-full.txt
+    const staticLlms = staticFile("llms.txt");
+    const llmsGen = gen([/(^|\/)app\/llms\.txt\/route\.(ts|js)$/, /(^|\/)src\/pages\/llms\.txt\.(ts|js)$/, /(^|\/)src\/routes\/llms\.txt\/\+server\.(ts|js)$/, /(^|\/)server\/routes\/llms\.txt\.(ts|js)$/]) ??
+        fromRoute("llms.txt", ["llms", "llmsTxt", "getLlms"]) ??
+        list(p, /(^|\/)scripts\/[^/]*llms[^/]*\.(m?[jt]s|cjs|py)$/i)[0];
+    const llms = staticLlms ? { exists: true, path: staticLlms, generated: false } : llmsGen ? { exists: true, path: llmsGen, generated: true } : { exists: false };
+    const staticLlmsFull = staticFile("llms-full.txt");
+    const llmsFullGen = gen([/(^|\/)app\/llms-full\.txt\/route\.(ts|js)$/, /(^|\/)src\/pages\/llms-full\.txt\.(ts|js)$/, /(^|\/)src\/routes\/llms-full\.txt\/\+server\.(ts|js)$/]) ??
+        fromRoute("llms-full.txt", ["llmsFull", "llmsFullTxt"]);
+    const llmsFull = staticLlmsFull ? { exists: true, path: staticLlmsFull, generated: false } : llmsFullGen ? { exists: true, path: llmsFullGen, generated: true } : { exists: false };
+    // 404 page
+    const nf = gen([/(^|\/)(src\/)?app\/not-found\.(tsx|jsx|js|ts)$/, /(^|\/)(src\/)?pages\/404\.(tsx|jsx|js|ts|astro|md|mdx|vue)$/, /(^|\/)(app\/)?error\.vue$/, /(^|\/)src\/routes\/\+error\.svelte$/, /(^|\/)app\/routes\/\$\.(tsx|jsx)$/, /(^|\/)layouts\/404\.html$/]) ??
+        first(p, [`${sd}404.html`, `${pre}404.html`, "404.html", "404.md"]);
+    let notFound = nf ? { exists: true, path: nf, generated: !/\.html?$/.test(nf) } : { exists: false };
+    if (!nf) {
+        const custom = code.find((f) => /\.(m?[jt]sx?|cjs)$/.test(f.rel) &&
+            !/(^|\/)(dev[-_.]?server|dev|mock|test)[^/]*\.[a-z]+$/i.test(f.rel) &&
+            /(status:\s*404|HttpError\(\s*404|\.status\(\s*404\s*\)|,\s*\{\s*status:\s*404|notFound\(\))/.test(f.text) &&
+            /not[\s_-]?found/i.test(f.text));
+        if (custom)
+            notFound = { exists: true, path: custom.rel, generated: true };
+        else if (rs.catchAll.length)
+            notFound = { exists: true, path: `${rs.catchAll[0].file}:${rs.catchAll[0].line} (client-side catch-all)`, generated: true };
+    }
+    // Static-file analysis
+    if (staticRobots)
+        findings.push(...staticRobotsFindings(staticRobots, text(p, staticRobots)));
+    if (staticSitemap) {
+        const sm = parseSitemapXml(text(p, staticSitemap));
+        const st = lastmodStats(sm.entries);
+        if (st.identical) {
+            findings.push(F("scan-sitemap-fake-lastmod", "medium", "read", "P4", `${staticSitemap}: every URL has the same lastmod (${st.commonDay})`, "lastmod should move only when a page's words change; one date everywhere says the dates mean nothing (book mistake #1).", { files: [staticSitemap], fix: "Generate the sitemap from page data with a per-page updated date.", accept: "lastmod differs between pages and only moves when a page's text changes" }));
+        }
+    }
+    if (staticLlms && (p.files.get(staticLlms)?.size ?? 0) > 50_000) {
+        findings.push(F("scan-llms-too-big", "low", "quote", "P8", `${staticLlms} is ${Math.round((p.files.get(staticLlms)?.size ?? 0) / 1024)} KB`, "Keep llms.txt short: product, pricing, the 10–20 key links; move long lists to llms-full.txt.", { files: [staticLlms], fix: "Trim to the essentials.", accept: "llms.txt under ~50 KB" }));
+    }
+    // Generated sitemap stamped with the build/request time
+    for (const g of sitemapGenerators) {
+        const t = text(p, g);
+        const stamp = /lastModified\s*:\s*new Date\(\s*\)|lastmod\s*:\s*new Date\(\s*\)|lastmod>\s*\$\{\s*new Date\(\s*\)|lastmod[^\n]{0,60}new Date\(\s*\)\.toISOString\(\)/.test(t);
+        const nextSitemapAuto = /next-sitemap\.config/.test(g) && !/autoLastmod\s*:\s*false/.test(t);
+        if (stamp || nextSitemapAuto) {
+            findings.push(F("scan-sitemap-build-lastmod", "medium", "read", "P4", "The sitemap stamps lastmod with the build/request time", `${g}: ${nextSitemapAuto ? "next-sitemap sets lastmod to the build time unless autoLastmod: false" : "lastmod = new Date()"}. Every deploy claims every page changed today — the book's mistake #1 (and it floods IndexNow).`, { files: [g], fix: "Use each page's real updated date (from content front-matter or the data file); omit lastmod if you don't have one.", accept: "lastmod only changes when a page's text changes" }));
+            break;
+        }
+    }
+    return { files: { robots, sitemap, llms, llmsFull, notFound }, staticRobots, staticSitemap, staticLlms, sitemapGenerators, findings };
+}
+function staticRobotsFindings(rel, body) {
+    const r = analyzeRobots(body, "https://example.invalid/robots.txt");
+    const out = [];
+    if (r.blocksAll) {
+        out.push(F("scan-robots-blocks-all", "critical", "read", "P4", `${rel} blocks the whole site (Disallow: / for *)`, "Every crawler without its own group is told to stay out.", { files: [rel], fix: "Disallow only app paths (/app/, /admin/, /api/) for *.", accept: "robots.txt has no Disallow: / for *" }));
+    }
+    const search = r.blockedTokens.filter((t) => ["search", "user", "search+training"].includes(crawlerByToken(t)?.purpose ?? ""));
+    if (search.length && !r.blocksAll) {
+        out.push(F("scan-robots-ai-search-blocked", "critical", "read", "P4", `${rel} blocks AI search/answer bots: ${search.join(", ")}`, "These bots decide whether you can appear in ChatGPT, Claude and Perplexity answers.", { files: [rel], fix: "Allow them by name (book appendix A).", accept: "OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User may fetch /" }));
+    }
+    const training = r.blockedTokens.filter((t) => !search.includes(t));
+    if (training.length && !r.blocksAll) {
+        out.push(F("scan-robots-training-blocked", "medium", "read", "P4", `${rel} blocks training bots: ${training.join(", ")}`, "A business decision — make sure it's deliberate. Blocking Google-Extended also removes Gemini grounding.", { files: [rel], fix: "Allow them unless you decided to opt out of training; keep Google-Extended allowed.", accept: "Every Disallow: / for an AI bot is written on purpose" }));
+    }
+    if (r.contentSignals.length) {
+        out.push(F("scan-robots-content-signal", "low", "read", "P4", `${rel} has a Content-Signal: line`, "Lighthouse and Search Console report it as an invalid robots.txt rule (book mistake #9).", { files: [rel], fix: "Write it as a comment: # Content signals: search=yes, ai-input=yes, ai-train=yes", accept: "Lighthouse shows robots.txt is valid" }));
+    }
+    for (const b of r.barePrefixes.filter((x) => x.pattern.replace(/^\//, "").length <= 2)) {
+        out.push(F("scan-robots-short-prefix", "low", "read", "P4", `${rel}: \`Disallow: ${b.pattern}\` is a bare prefix`, `It also blocks every path that starts with "${b.pattern}" (e.g. ${b.pattern}ools).`, { files: [rel], fix: `Use \`Disallow: ${b.pattern}/\`.`, accept: "No bare short prefixes" }));
+    }
+    if (!r.sitemaps.length) {
+        out.push(F("scan-robots-no-sitemap", "low", "read", "P4", `${rel} has no Sitemap line`, "", { files: [rel], fix: "Add `Sitemap: https://<domain>/sitemap.xml` at the end.", accept: "robots.txt ends with a Sitemap: line" }));
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Head tags
+export function detectHeadSource(p, det, rs) {
+    const code = codeFiles(p);
+    const fw = det.framework.name;
+    const findings = [];
+    const notes = [];
+    const central = code.find((f) => /(^|\/)(seo|metadata|site-?config|siteConfig|meta|head|BaseHead|SEO|Seo|Head)(\.config)?\.(ts|js|tsx|jsx|mjs|astro|svelte|vue)$/.test(f.rel) || /(^|\/)seo\//i.test(f.rel));
+    const count = (re) => code.filter((f) => re.test(f.text));
+    let info = { exists: false };
+    let setters = [];
+    if (fw === "next") {
+        setters = count(/export\s+const\s+metadata\s*[:=]|export\s+(async\s+)?function\s+generateMetadata|from\s+["']next\/head["']|<NextSeo|<DefaultSeo|next-seo/);
+        const pages = rs.routes.filter((r) => r.file && /(^|\/)page\.(tsx|jsx|ts|js|mdx)$/.test(r.file) && r.kind === "public" && !r.dynamic);
+        const withoutOwn = pages.filter((r) => !/export\s+const\s+metadata|generateMetadata/.test(text(p, r.file)));
+        if (pages.length >= 3 && withoutOwn.length / pages.length > 0.5) {
+            findings.push(F("scan-pages-inherit-title", "medium", "quote", "P3", `${withoutOwn.length} of ${pages.length} public pages have no own metadata`, "They inherit the root layout's title and description, so several URLs share the same title.", { files: withoutOwn.slice(0, 10).map((r) => r.file), fix: "Export metadata (title ~60 chars, one-sentence description, canonical, openGraph) per page from a single SEO source file.", accept: "Every public page has a unique title and description" }));
+        }
+    }
+    else if (["vite-react", "cra", "static-html"].includes(fw) && det.rendering.verdict === "csr") {
+        setters = count(/react-helmet|@unhead\/react|react-head\b|<Helmet\b|useHead\(|document\.title\s*=/);
+    }
+    else if (fw === "vite-vue" || fw === "nuxt") {
+        setters = count(/useHead\(|useSeoMeta\(|vue-meta|@unhead\/vue|@vueuse\/head|definePageMeta\(/);
+    }
+    else if (fw === "sveltekit" || fw === "vite-svelte") {
+        setters = count(/<svelte:head>/);
+    }
+    else if (fw === "astro") {
+        setters = count(/<title>|<SEO\b|astro-seo/).filter((f) => /\.astro$/.test(f.rel));
+    }
+    else if (fw === "remix" || fw === "react-router") {
+        setters = count(/export\s+(const|function)\s+meta\b/);
+    }
+    else if (fw === "gatsby") {
+        setters = count(/export\s+(const|function)\s+Head\b|react-helmet|<Seo\b|<SEO\b/);
+    }
+    else if (fw === "angular") {
+        setters = count(/\bTitle\b[\s\S]{0,200}setTitle\(|\bMeta\b[\s\S]{0,200}updateTag\(/);
+    }
+    else {
+        const staticPrefix = det.staticDir ? `${det.staticDir}/` : "\u0000";
+        const generated = fw === "custom-server" && det.rendering.verdict === "ssg";
+        setters = count(/<title>|<title>\$\{|<title>\{|document\.title\s*=|<meta\s+name=["']description["']/).filter((f) => !(generated && f.rel.startsWith(staticPrefix) && /\.html?$/.test(f.rel)));
+        if (generated) {
+            const gen = codeFiles(p, { includeScripts: true }).find((f) => /<title>\$\{/.test(f.text) && /writeFile|outputFile/.test(f.text));
+            if (gen)
+                setters.unshift(gen);
+        }
+    }
+    if (central) {
+        info = { exists: true, path: central.rel, generated: true };
+        notes.push(`single SEO source: ${central.rel}`);
+    }
+    else if (setters.length) {
+        info = { exists: true, path: setters[0].rel, generated: !/\.html?$/.test(setters[0].rel) };
+        if (setters.length > 5 && fw !== "static-html") {
+            findings.push(F("scan-head-scattered", "medium", "quote", "P3", `Titles and descriptions are set in ${setters.length} different files`, "Head tags written by hand in many files get copied and go stale; the assistant's one sentence comes from the description.", { files: setters.slice(0, 10).map((f) => f.rel), fix: "Generate every page's title, description, canonical and og tags from one file (e.g. src/seo/pages.ts) and add a test that fails on duplicates (P3).", accept: "Each page's head comes from one file; the duplicate test passes" }));
+        }
+    }
+    if (!info.exists) {
+        const spa = det.rendering.verdict === "csr";
+        findings.push(F("scan-head-missing", "high", "quote", "P3", spa ? "Every URL serves index.html's title and description" : "No per-page title/description found", spa
+            ? "The single-page app has no per-route head management, so every route shares one title and description (and crawlers without JavaScript would not see per-route tags anyway)."
+            : "No code sets page titles or meta descriptions.", { files: det.spa.indexHtml ? [det.spa.indexHtml] : [], fix: "Create one SEO source (title ~60 chars, one-sentence description, canonical, og:*) per page and render it into the server HTML (P3; for SPAs together with P2 prerendering).", accept: "Each public page has a unique title and description in the server HTML" }));
+    }
+    // canonical / og:image anywhere in the code
+    const all = [...codeFiles(p), ...[...p.files.values()].filter((f) => f.text && /\.html?$/.test(f.rel))];
+    const anyCanonical = all.some((f) => /rel=["'{]?\s*["']?canonical|canonical\s*[:=]|alternates\s*:\s*\{[^}]*canonical/.test(f.text));
+    const anyOg = all.some((f) => /og:image|openGraph\s*:\s*\{|opengraph-image|twitter:image|ogImage/.test(f.text)) || list(p, /(^|\/)opengraph-image\.(png|jpg|tsx|jsx|ts|js)$/).length > 0;
+    if (!anyCanonical) {
+        findings.push(F("scan-no-canonical", "medium", "read", "P3", "No canonical tags in the code", "Without a self-referencing canonical, Google picks the preferred URL itself (it may pick a preview copy or a www twin).", { fix: "Emit one absolute, self-referencing canonical per page from the SEO source.", accept: "Every page has exactly one absolute canonical to itself" }));
+    }
+    if (!anyOg) {
+        findings.push(F("scan-no-og-image", "medium", "quote", "P3", "No og:image in the code", "Shared links show an empty card; Google also uses og:image as a thumbnail source (the book's katman.pro had 0/40).", { fix: "Add a 1200×630 og:image per page (absolute URL) with og:image:alt, og:title, og:description, og:url and twitter:card.", accept: "Every page has an og:image that opens in the browser" }));
+    }
+    return { info, findings, notes };
+}
+// ---------------------------------------------------------------------------------------------------------------
+// JSON-LD, i18n, analytics
+export function jsonLdTypes(p) {
+    const out = new Set();
+    for (const f of p.files.values()) {
+        if (!f.text || f.minified || !/\.(m?[jt]sx?|cjs|astro|vue|svelte|html?|json|njk|liquid|php)$/.test(f.rel))
+            continue;
+        if (/(^|\/)(package(-lock)?|tsconfig|components)\.json$/.test(f.rel))
+            continue;
+        for (const m of f.text.matchAll(/["']@type["']\s*:\s*["'`]([A-Z][A-Za-z0-9]{2,40})["'`]/g))
+            out.add(m[1]);
+    }
+    return [...out].sort();
+}
+const I18N_LIBS = [
+    ["next-intl", "next-intl"], ["next-i18next", "next-i18next"], ["next-translate", "next-translate"], ["react-i18next", "react-i18next"],
+    ["i18next", "i18next"], ["@nuxtjs/i18n", "@nuxtjs/i18n"], ["vue-i18n", "vue-i18n"], ["svelte-i18n", "svelte-i18n"],
+    ["@inlang/paraglide-js", "paraglide"], ["@inlang/paraglide-sveltekit", "paraglide"], ["astro-i18next", "astro-i18next"],
+    ["@lingui/core", "lingui"], ["react-intl", "react-intl"], ["typesafe-i18n", "typesafe-i18n"], ["@angular/localize", "@angular/localize"],
+];
+const LOCALE = /^[a-z]{2}(-[A-Z]{2}|-[a-z]{2,4})?$/;
+export function detectI18n(p, det) {
+    const deps = det.pkg?.deps ?? {};
+    const code = codeFiles(p);
+    let library = I18N_LIBS.find(([d]) => d in deps)?.[1];
+    const locales = new Set();
+    for (const f of code.concat([...p.files.values()].filter((x) => x.text && /(^|\/)(next|astro|nuxt)\.config\.[mc]?[jt]s$/.test(x.rel)))) {
+        for (const m of f.text.matchAll(/\b(?:LANGS|LOCALES|SITE_LANGS|locales|supportedLngs|SUPPORTED_LOCALES|supportedLocales|SUPPORTED_LANGS)\s*[:=]\s*\[([^\]]{1,600})\]/g)) {
+            for (const s of m[1].matchAll(/["'`]([a-zA-Z-]{2,10})["'`]/g))
+                if (LOCALE.test(s[1]))
+                    locales.add(s[1]);
+            for (const s of m[1].matchAll(/code\s*:\s*["'`]([a-zA-Z-]{2,10})["'`]/g))
+                if (LOCALE.test(s[1]))
+                    locales.add(s[1]);
+        }
+    }
+    for (const f of p.files.keys()) {
+        const m = f.match(/(?:^|\/)(?:locales|messages|i18n|lang|langs|translations|dictionaries)\/([a-z]{2}(?:[-_][A-Za-z]{2})?)(?:\.(?:json|ya?ml|ts|js)|\/)/);
+        if (m && LOCALE.test(m[1].replace("_", "-")))
+            locales.add(m[1].replace("_", "-"));
+    }
+    if (!library && code.some((f) => /(^|\/)i18n\.(m?[jt]s)$/.test(f.rel)))
+        library = "custom";
+    // <html lang>
+    let htmlLang;
+    for (const f of [...p.files.values()].filter((x) => x.text && /(^|\/)(index\.html|app\.html|_document\.[jt]sx?|layout\.[jt]sx?|root\.[jt]sx|.*Layout\.astro|layout\.astro|nuxt\.config\.[jt]s|shells?\.[jt]s|layout\.m?js)$/.test(x.rel))) {
+        const t = f.text;
+        const m = t.match(/<[Hh]tml[^>]*\blang\s*=\s*\{?\s*["'`]([a-zA-Z-]{2,10})["'`]/) ?? t.match(/htmlAttrs\s*:\s*\{[^}]*lang\s*:\s*["'`]([a-zA-Z-]{2,10})["'`]/);
+        if (m) {
+            htmlLang = m[1];
+            break;
+        }
+        if (/<[Hh]tml[^>]*\blang\s*=\s*(\{|["'`]\$\{)/.test(t)) {
+            htmlLang = "dynamic";
+            break;
+        }
+    }
+    const detected = !!library || locales.size > 1;
+    const findings = [];
+    if (locales.size > 1) {
+        const hreflang = code.some((f) => /hreflang|hrefLang|alternates\s*:\s*\{[\s\S]{0,200}languages|xhtml:link/.test(f.text));
+        if (!hreflang) {
+            findings.push(F("scan-i18n-no-hreflang", "medium", "read", "P3", `${locales.size} languages but no hreflang in the code`, `Locales: ${[...locales].join(", ")}. Without reciprocal hreflang (plus x-default), search engines can't match each language version to its audience.`, { fix: "Emit hreflang for every translated version (itself included) and x-default, only for pages that really exist in that language.", accept: "Every hreflang target returns 200 and links back" }));
+        }
+    }
+    if (!htmlLang && det.framework.name !== "unknown") {
+        findings.push(F("scan-no-html-lang", "low", "read", "P3", "No <html lang> found", "Set the page language on the <html> element.", { fix: "Add lang to <html> (per language on multilingual sites).", accept: "Every page declares its language" }));
+    }
+    return { i18n: { detected, library, locales: [...locales].sort(), htmlLang }, findings };
+}
+const ANALYTICS = [
+    [/googletagmanager\.com\/gtag\/js|gtag\(\s*['"]config['"]|@next\/third-parties\/google|react-ga4|vue-gtag|['"]G-[A-Z0-9]{6,}['"]/, "google-analytics"],
+    [/googletagmanager\.com\/gtm\.js|['"]GTM-[A-Z0-9]{4,}['"]/, "gtm"],
+    [/plausible\.io\/js|next-plausible|plausible-tracker/, "plausible"],
+    [/usefathom\.com|fathom-client/, "fathom"],
+    [/posthog-js|posthog\.init|posthog-node/, "posthog"],
+    [/cloud\.umami\.is|umami\.is\/script|data-website-id=/, "umami"],
+    [/@vercel\/analytics/, "vercel-analytics"],
+    [/static\.cloudflareinsights\.com\/beacon/, "cloudflare-web-analytics"],
+    [/simpleanalyticscdn\.com/, "simple-analytics"],
+    [/mixpanel-browser|mixpanel\.init\(/, "mixpanel"],
+    [/@amplitude\/|amplitude\.init\(/, "amplitude"],
+    [/@segment\/analytics|analytics\.load\(/, "segment"],
+    [/matomo\.js|_paq\.push/, "matomo"],
+    [/clarity\.ms|@microsoft\/clarity/, "microsoft-clarity"],
+    [/static\.hotjar\.com/, "hotjar"],
+    [/connect\.facebook\.net\/[^"']*fbevents|fbq\(\s*['"]init/, "meta-pixel"],
+    [/navigator\.sendBeacon\(|\/api\/(collect|track|events?|analytics|beacon|hit)\b/, "custom"],
+];
+const ANALYTICS_DEPS = {
+    "@vercel/analytics": "vercel-analytics", "posthog-js": "posthog", "posthog-node": "posthog", "mixpanel-browser": "mixpanel",
+    "@amplitude/analytics-browser": "amplitude", "react-ga4": "google-analytics", "vue-gtag": "google-analytics", "next-plausible": "plausible",
+    "plausible-tracker": "plausible", "fathom-client": "fathom", "@segment/analytics-next": "segment", "@microsoft/clarity": "microsoft-clarity",
+    "@next/third-parties": "google-analytics", "@umami/node": "umami", "nuxt-gtag": "google-analytics", "@nuxtjs/plausible": "plausible",
+};
+export function detectAnalytics(p, deps = {}) {
+    // Client scripts in public/ count here (tracking snippets live there), but not minified bundles.
+    const publicJs = [...p.files.values()].filter((f) => f.text && !f.minified && /(^|\/)(public|static)\/.*\.m?js$/.test(f.rel));
+    const code = [...codeFiles(p, { includeScripts: true }), ...publicJs].filter((f) => !/\.(md|mdx)$/.test(f.rel));
+    const providers = new Set();
+    for (const d of Object.keys(deps))
+        if (ANALYTICS_DEPS[d])
+            providers.add(ANALYTICS_DEPS[d]);
+    for (const f of code)
+        for (const [re, name] of ANALYTICS)
+            if (re.test(f.text))
+                providers.add(name);
+    if (code.some((f) => /(^|\/)(analytics|track|tracking|telemetry|beacon|metrics|visits?)\.(m?[jt]s|cjs)$/.test(f.rel) && /utm_source|referr|pageview|page_view|session/i.test(f.text))) {
+        providers.delete("custom");
+        providers.add("first-party");
+    }
+    const tracksAiChannel = code.some((f) => /chatgpt\.com|chat\.openai\.com|perplexity\.ai|claude\.ai|gemini\.google\.com/.test(f.text) && /referr|utm_source|searchParams|channel/i.test(f.text));
+    const botCounting = code.some((f) => /GPTBot|OAI-SearchBot|ClaudeBot|PerplexityBot/.test(f.text) && /user-agent|userAgent|headers\.get\(\s*['"]user-agent/i.test(f.text));
+    const findings = [];
+    if (!providers.size) {
+        findings.push(F("scan-no-analytics", "low", "measure", "P13", "No analytics found", "You can't see visitors from ChatGPT (or anywhere) without some analytics.", { fix: "Add privacy-friendly analytics with an AI channel (referrer chatgpt.com OR utm_source=chatgpt.com) and a human filter (P13).", accept: "The ChatGPT channel also counts visits without a referrer; bot counts come from the server" }));
+    }
+    if (!tracksAiChannel) {
+        findings.push(F("scan-no-ai-channel", "medium", "measure", "P13", "ChatGPT visits aren't tracked as their own channel", "Most ChatGPT visitors send no referrer (79% on the book's example site) but carry utm_source=chatgpt.com; referrer-only analytics files them under \"direct\".", { fix: "Classify a session as AI when the referrer is chatgpt.com/perplexity.ai/claude.ai/gemini.google.com… OR the landing URL has utm_source=chatgpt.com (P13).", accept: "ChatGPT channel counts referrer-less visits; bot counts come from the server" }));
+    }
+    if (!botCounting) {
+        findings.push(F("scan-no-bot-counting", "low", "measure", "P13", "AI bots aren't counted on the server", "Bots don't run analytics JavaScript; only server/edge logs show whether GPTBot, OAI-SearchBot, ClaudeBot or PerplexityBot read your pages.", { fix: "Count requests per bot user agent per day in middleware/the Worker (katman_generate can emit a snippet).", accept: "Daily bot counts per crawler are visible (server-side)" }));
+    }
+    return { analytics: { providers: [...providers].sort(), tracksAiChannel }, findings, botCounting };
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Links (onClick navigation) and images
+const NAVISH = /(nav|menu|header|footer|sidebar|card|navbar|topbar|layout|hero|tile|grid|list|link|breadcrumb)/i;
+export function detectLinksAndImages(p) {
+    const code = codeFiles(p).filter((f) => /\.(jsx|tsx|vue|svelte|astro|html?|js|ts)$/.test(f.rel));
+    const findings = [];
+    const navHits = [];
+    let anchors = 0;
+    for (const f of code) {
+        const t = f.text;
+        anchors += (t.match(/<a\s[^>]*href=|<Link\b|<NavLink\b|<NuxtLink\b|<router-link\b|<RouterLink\b/g) ?? []).length;
+        for (const m of t.matchAll(/(?:onClick|@click(?:\.\w+)?|on:click|onclick|\(click\))\s*=\s*[{"']/g)) {
+            const window = t.slice(m.index, m.index + 220);
+            const nav = window.match(/\b(navigate|router\.(?:push|replace)|history\.push|goto|location\.assign|\$router\.push|window\.location(?:\.href)?\s*=)\s*\(?\s*(["'`]([^"'`]*)["'`])?/);
+            if (!nav)
+                continue;
+            navHits.push({ file: f.rel, line: lineOf(t, m.index), target: nav[3], navish: NAVISH.test(f.rel.split("/").pop() ?? "") });
+        }
+    }
+    const publicHits = navHits.filter((h) => !h.target || (h.target.startsWith("/") && ["public", "legal"].includes(classifyPath(h.target))));
+    if (publicHits.length) {
+        const inNav = publicHits.filter((h) => h.navish);
+        findings.push(F("scan-onclick-nav", inNav.length ? "high" : "medium", "read", "P10", `${publicHits.length} onClick navigation${publicHits.length === 1 ? "" : "s"} to public pages${inNav.length ? ` (${inNav.length} in nav/menu/card components)` : ""}`, `Crawlers follow only <a href> links; a div/button that navigates in onClick is invisible to them (book trap #5). ${anchors} <a>/<Link> elements found for comparison.`, { files: [...new Set(publicHits.map((h) => `${h.file}:${h.line}`))].slice(0, 10), evidence: publicHits.slice(0, 5).map((h) => `${h.file}:${h.line} → ${h.target ?? "(computed)"}`), fix: "Use <a href> / <Link href> for every navigation to a public page (menus, cards, footers); keep onClick only for actions.", accept: "Menus and cards use <a href>; `curl -s https://site/ | grep -o '<a [^>]*href=\"/' | wc -l` ≥ number of menu items" }));
+    }
+    // Images
+    let total = 0;
+    let noAlt = 0;
+    let noSize = 0;
+    const altFiles = new Set();
+    const sizeFiles = new Set();
+    for (const f of code) {
+        for (const m of f.text.matchAll(/<img\b[^>]*>/gi)) {
+            total++;
+            const tag = m[0];
+            if (!/\balt\s*=/.test(tag) && !/\{\s*\.\.\.\w+\s*\}/.test(tag)) {
+                noAlt++;
+                altFiles.add(`${f.rel}:${lineOf(f.text, m.index)}`);
+            }
+            if (!/\bwidth\s*=/.test(tag) || !/\bheight\s*=/.test(tag)) {
+                noSize++;
+                sizeFiles.add(f.rel);
+            }
+        }
+    }
+    if (noAlt || noSize > 3) {
+        findings.push(F("scan-images", noAlt ? "low" : "info", "quote", "P9", `${noAlt} <img> without alt, ${noSize} without width/height (of ${total})`, "Informative images need a short alt (decorative ones alt=\"\"); width and height prevent layout shift (CLS).", { files: [...altFiles].slice(0, 10), evidence: [...sizeFiles].slice(0, 5).map((f) => `no size: ${f}`), fix: "Add alt to every <img>, width/height (or aspect-ratio), loading=\"lazy\" below the fold; keep images under 200 KB (WebP/AVIF).", accept: "No <img> without alt; no image over 200 KB; CLS < 0.1" }));
+    }
+    return findings;
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Site-wide noindex and header pitfalls
+export function detectIndexingPitfalls(p) {
+    const findings = [];
+    const roots = [...p.files.values()].filter((f) => f.text && /(^|\/)(index\.html|app\.html|(src\/)?app\/layout\.[jt]sx?|pages\/_(document|app)\.[jt]sx?|app\/root\.[jt]sx?|nuxt\.config\.[jt]s|src\/layouts\/[^/]*\.astro|layout\.m?js)$/.test(f.rel));
+    for (const f of roots) {
+        const t = f.text;
+        const m = t.match(/<meta[^>]+name=["']robots["'][^>]+content=["'][^"']*noindex|robots\s*:\s*\{[^}]*\bindex\s*:\s*false|robots\s*:\s*["'`]noindex/);
+        if (!m)
+            continue;
+        const before = t.slice(Math.max(0, (m.index ?? 0) - 90), m.index ?? 0);
+        if (/\?\s*$|\?\s*['"`(]|&&\s*[('"`<]?\s*$|\$\{[^}]*$|if\s*\(|noindex\s*\?|:\s*$/.test(before))
+            continue; // conditional
+        findings.push(F("scan-sitewide-noindex", "critical", "read", "P4", `Site-wide noindex in ${f.rel}`, "The root layout / index.html tells search engines not to index any page (often left over from staging).", { files: [`${f.rel}:${lineOf(t, m.index ?? 0)}`], fix: "Remove the noindex from the root; add noindex only to login/checkout/thank-you/search pages.", accept: "Public pages have no noindex (katman_audit_site shows 0 noindex pages in the sitemap)" }));
+    }
+    const headerFiles = [...p.files.values()].filter((f) => f.text && /(^|\/)(_headers|vercel\.json|netlify\.toml)$/.test(f.rel));
+    for (const f of headerFiles) {
+        const t = f.text;
+        if (/x-robots-tag["']?\s*[:=]\s*["']?[^\n"']*noindex/i.test(t) && /(^\/\*\s*$|"source"\s*:\s*"\/\(\.\*\)"|for\s*=\s*["']\/\*["'])/m.test(t)) {
+            findings.push(F("scan-header-noindex", "critical", "read", "P4", `${f.rel} sends X-Robots-Tag: noindex`, "A header rule adds noindex (check whether it applies to all paths on the production domain).", { files: [f.rel], fix: "Scope X-Robots-Tag: noindex to preview hosts or private paths only.", accept: "Production pages answer without X-Robots-Tag: noindex" }));
+        }
+        if (/no-transform/i.test(t)) {
+            findings.push(F("scan-no-transform", "medium", "read", "P14", `${f.rel} sets Cache-Control: no-transform`, "no-transform stops Cloudflare (and other CDNs) from compressing HTML — the book's example shipped pages ~4× larger until it was found (mistake #10).", { files: [f.rel], fix: "Remove no-transform from HTML responses (or compress in your own code).", accept: "curl -s -o /dev/null -D - -H 'Accept-Encoding: br, gzip' https://site/ | grep -i content-encoding shows br or gzip" }));
+        }
+    }
+    const selfCompress = codeFiles(p).some((f) => /CompressionStream|['"]Content-Encoding['"]\s*,\s*[a-z'"]|set\(\s*['"]content-encoding['"]/i.test(f.text));
+    for (const f of codeFiles(p)) {
+        if (selfCompress)
+            break;
+        if (/no-transform/.test(f.text) && /cache-control/i.test(f.text) && /\.(m?[jt]s|cjs)$/.test(f.rel)) {
+            findings.push(F("scan-no-transform-code", "medium", "read", "P14", `${f.rel} sends Cache-Control: no-transform`, "no-transform can silently disable CDN compression of HTML (the book's mistake #10).", { files: [f.rel], fix: "Drop no-transform for HTML, or compress in the Worker yourself; re-check headers after every fix.", accept: "HTML responses carry Content-Encoding: br or gzip" }));
+            break;
+        }
+    }
+    return findings;
+}

package/dist/tools/method/cli.js

@@ -0,0 +1,42 @@
+// CLI commands of the method module: `katman-mcp report` and `katman-mcp plan` (same core as the MCP tools).
+import path from "node:path";
+import { parseArgs } from "../../cli.js";
+import { runReport } from "./core/report.js";
+import { runPlan } from "./core/plan.js";
+function rootOf(flags, positional) {
+    const r = typeof flags.root === "string" ? flags.root : positional[0];
+    return path.resolve(r ?? process.cwd());
+}
+function langOf(flags) {
+    return flags.lang === "tr" || flags.lang === "en" ? flags.lang : undefined;
+}
+export const commands = {
+    report: async (args) => {
+        const { _, flags } = parseArgs(args);
+        try {
+            const out = await runReport(rootOf(flags, _), { lang: langOf(flags) });
+            console.log(out.text);
+            return out.isError ? 1 : 0;
+        }
+        catch (e) {
+            console.error(`katman report: ${e instanceof Error ? e.message : String(e)}`);
+            return 1;
+        }
+    },
+    plan: async (args) => {
+        const { _, flags } = parseArgs(args);
+        try {
+            const out = await runPlan(rootOf(flags, _), { lang: langOf(flags) });
+            console.log(out.text);
+            return out.isError ? 1 : 0;
+        }
+        catch (e) {
+            console.error(`katman plan: ${e instanceof Error ? e.message : String(e)}`);
+            return 1;
+        }
+    },
+};
+export const help = [
+    "  katman-mcp plan [--root .] [--lang en|tr]     build .katman/plan.md: page plan + 30-day task list (keeps your ticks)",
+    "  katman-mcp report [--root .] [--lang en|tr]   write .katman/report.md: research, audit, AI visibility, plan, next 7 days",
+];