katman-mcp 0.1.0

package/dist/tools/method/core/extract.js

@@ -0,0 +1,435 @@
+// Rough visible-text extraction for katman_lint_content: Markdown/MDX, HTML, Astro, Vue, Svelte, JSX/TSX and
+// fetched pages. "Rough" on purpose: it finds headings, paragraphs, links, FAQ questions, title/description.
+import { parse, HTMLElement as HTMLElementClass } from "node-html-parser";
+import { countWords, splitSentences } from "../lib/text.js";
+const DATE_WORDS_RE = /(last\s+updated|updated\s*(on|:)?|güncellen\w*|son\s+güncelleme|dateModified|lastmod|modified\s*:|<time\b[^>]*datetime=)/i;
+const FAQ_HEAD_RE = /\b(faq|frequently asked|questions|sss|sık sorulan|sorular)\b/i;
+function clean(s) {
+    return s.replace(/\s+/g, " ").trim();
+}
+function questionish(s) {
+    const x = s.trim();
+    return x.length > 8 && x.length < 200 && /\?$/.test(x);
+}
+function textOf(el) {
+    return el ? clean(el.text) : "";
+}
+// ---------------------------------------------------------------------------
+// HTML
+export function extractHtml(html, label, source = "file") {
+    const root = parse(html, { comment: false, blockTextElements: { script: true, style: true, noscript: true, pre: true } });
+    const metas = root.querySelectorAll("meta");
+    const metaVal = (name) => metas.find((m) => (m.getAttribute("name") ?? "").toLowerCase() === name)?.getAttribute("content") ?? undefined;
+    const jsonLd = [];
+    let jsonLdErrors = 0;
+    for (const s of root.querySelectorAll("script")) {
+        if (!/ld\+json/i.test(s.getAttribute("type") ?? ""))
+            continue;
+        try {
+            jsonLd.push(JSON.parse(s.rawText));
+        }
+        catch {
+            jsonLdErrors++;
+        }
+    }
+    const title = textOf(root.querySelector("title")) || undefined;
+    const lang = root.querySelector("html")?.getAttribute("lang") ?? undefined;
+    for (const n of root.querySelectorAll("script, style, noscript, template, svg"))
+        n.remove();
+    const body = root.querySelector("body") ?? root;
+    const allLinks = body.querySelectorAll("a").map((a) => a.getAttribute("href") ?? "").filter(Boolean);
+    let main = root.querySelector("main") ?? root.querySelector("article");
+    let contentLinks;
+    if (main)
+        contentLinks = main.querySelectorAll("a").map((a) => a.getAttribute("href") ?? "").filter(Boolean);
+    if (!main) {
+        for (const n of body.querySelectorAll("header, nav, footer"))
+            n.remove();
+        main = body;
+        contentLinks = main.querySelectorAll("a").map((a) => a.getAttribute("href") ?? "").filter(Boolean);
+    }
+    const headings = main.querySelectorAll("h1, h2, h3, h4, h5, h6").map((h) => ({ level: Number(h.tagName.slice(1)), text: textOf(h) }));
+    const h1 = headings.filter((h) => h.level === 1).map((h) => h.text).filter(Boolean);
+    if (!h1.length) {
+        const outerH1 = root.querySelectorAll("h1").map((h) => textOf(h)).filter(Boolean);
+        h1.push(...outerH1);
+    }
+    // blocks in document order, remembering where the first H1 is
+    const blocks = [];
+    let afterH1 = 0;
+    let seenH1 = false;
+    const walk = (el) => {
+        for (const child of el.childNodes) {
+            if (!(child instanceof HTMLElementClass))
+                continue;
+            const c = child;
+            const tag = (c.tagName ?? "").toLowerCase();
+            if (tag === "h1") {
+                if (!seenH1) {
+                    seenH1 = true;
+                    afterH1 = blocks.length;
+                }
+                continue;
+            }
+            if (["p", "li", "dd", "blockquote", "summary", "dt", "figcaption", "td"].includes(tag)) {
+                const t = textOf(c);
+                if (t)
+                    blocks.push(t);
+                continue;
+            }
+            if (/^h[2-6]$/.test(tag))
+                continue;
+            walk(c);
+        }
+    };
+    walk(main);
+    const faqQuestions = new Set();
+    let inFaq = false;
+    for (const h of headings) {
+        if (h.level <= 2)
+            inFaq = FAQ_HEAD_RE.test(h.text) || (inFaq && questionish(h.text));
+        if (questionish(h.text) && (inFaq || h.level >= 2))
+            faqQuestions.add(h.text);
+    }
+    for (const n of main.querySelectorAll("summary, dt")) {
+        const t = textOf(n);
+        if (questionish(t))
+            faqQuestions.add(t);
+    }
+    for (const j of jsonLd) {
+        const items = Array.isArray(j) ? j : [j];
+        for (const it of items) {
+            const graph = Array.isArray(it?.["@graph"]) ? it["@graph"] : [it];
+            for (const g of graph) {
+                if (g && g["@type"] === "FAQPage" && Array.isArray(g.mainEntity)) {
+                    for (const q of g.mainEntity)
+                        if (typeof q?.name === "string")
+                            faqQuestions.add(clean(q.name));
+                }
+            }
+        }
+    }
+    const text = textOf(main);
+    const fullText = textOf(body);
+    return {
+        source,
+        kind: "html",
+        label,
+        raw: html,
+        ...(title ? { title } : {}),
+        ...(metaVal("description") !== undefined ? { description: clean(metaVal("description")) } : {}),
+        ...(metaVal("robots") ? { robotsMeta: metaVal("robots") } : {}),
+        ...(lang ? { lang } : {}),
+        h1,
+        headings,
+        blocks,
+        afterH1,
+        text,
+        links: allLinks,
+        contentLinks,
+        faqQuestions: [...faqQuestions],
+        jsonLd,
+        jsonLdErrors,
+        hasDate: DATE_WORDS_RE.test(html) || jsonLd.some((j) => JSON.stringify(j).includes("dateModified")),
+        wordsJsOff: countWords(fullText),
+        notes: [],
+    };
+}
+// ---------------------------------------------------------------------------
+// Markdown / MDX / plain text
+function frontmatter(src) {
+    const m = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/.exec(src);
+    if (!m)
+        return { data: {}, body: src };
+    const data = {};
+    for (const line of m[1].split(/\r?\n/)) {
+        const kv = /^([A-Za-z_][\w-]*)\s*:\s*(.*)$/.exec(line);
+        if (kv)
+            data[kv[1].toLowerCase()] = kv[2].replace(/^["']|["']$/g, "").trim();
+    }
+    return { data, body: src.slice(m[0].length) };
+}
+function stripMd(s) {
+    return clean(s
+        .replace(/!\[[^\]]*\]\([^)]*\)/g, "")
+        .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/[*_`~]+/g, "")
+        .replace(/^>\s?/gm, ""));
+}
+export function extractMarkdown(src, label, source = "file") {
+    const { data, body: b0 } = frontmatter(src);
+    const body = b0
+        .replace(/^\s*(import|export)\s.*$/gm, "")
+        .replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/```[\s\S]*?```/g, "");
+    const headings = [];
+    const blocks = [];
+    let afterH1 = 0;
+    let seenH1 = false;
+    const links = [...body.matchAll(/\[[^\]]*\]\(([^)\s]+)[^)]*\)/g)].map((m) => m[1]).concat([...body.matchAll(/<a\s[^>]*href=["']([^"']+)["']/gi)].map((m) => m[1]));
+    const faqQuestions = new Set();
+    let para = [];
+    let inFaq = false;
+    const flush = () => {
+        if (para.length) {
+            const t = stripMd(para.join(" "));
+            if (t)
+                blocks.push(t);
+            para = [];
+        }
+    };
+    for (const line of body.split(/\r?\n/)) {
+        const h = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);
+        if (h) {
+            flush();
+            const text = stripMd(h[2]);
+            const level = h[1].length;
+            headings.push({ level, text });
+            if (level === 1 && !seenH1) {
+                seenH1 = true;
+                afterH1 = blocks.length;
+            }
+            if (level <= 2)
+                inFaq = FAQ_HEAD_RE.test(text) || (inFaq && questionish(text));
+            if (questionish(text))
+                faqQuestions.add(text);
+            continue;
+        }
+        const li = /^\s*(?:[-*+]|\d+[.)])\s+(.*)$/.exec(line);
+        if (li) {
+            flush();
+            const t = stripMd(li[1]);
+            if (t)
+                blocks.push(t);
+            if (inFaq && questionish(t))
+                faqQuestions.add(t);
+            continue;
+        }
+        const bold = /^\s*\*\*(.+\?)\*\*\s*$/.exec(line);
+        if (bold) {
+            flush();
+            faqQuestions.add(stripMd(bold[1]));
+            continue;
+        }
+        if (!line.trim()) {
+            flush();
+            continue;
+        }
+        if (/^\s*\|/.test(line)) {
+            flush();
+            const cells = line.split("|").map((c) => stripMd(c)).filter((c) => c && !/^-+$/.test(c));
+            if (cells.length)
+                blocks.push(cells.join(" · "));
+            continue;
+        }
+        para.push(line);
+    }
+    flush();
+    let h1 = headings.filter((x) => x.level === 1).map((x) => x.text);
+    const notes = [];
+    if (!h1.length && data.title) {
+        h1 = [data.title];
+        notes.push("No # heading: the frontmatter title is treated as the H1.");
+    }
+    const text = blocks.join("\n");
+    return {
+        source,
+        kind: source === "text" ? "text" : "markdown",
+        label,
+        raw: src,
+        ...(data.title ? { title: data.title } : {}),
+        ...(data.description ? { description: data.description } : {}),
+        h1,
+        headings,
+        blocks,
+        afterH1,
+        text,
+        links,
+        faqQuestions: [...faqQuestions],
+        jsonLd: [],
+        jsonLdErrors: 0,
+        hasDate: DATE_WORDS_RE.test(src) || !!(data.updated || data.lastmod || data.datemodified || data.modified || data.updatedate),
+        notes,
+    };
+}
+// ---------------------------------------------------------------------------
+// Astro / Vue / Svelte single-file components
+function replaceExpressions(s) {
+    // {{ vue }} and { svelte / astro } expressions: keep plain string literals, drop the rest
+    return s
+        .replace(/\{\{[\s\S]*?\}\}/g, " ")
+        .replace(/\{\s*(["'`])((?:\\.|(?!\1)[^\\])*)\1\s*\}/g, (_m, _q, str) => (str.includes("${") ? " " : str))
+        .replace(/\{[^{}]*\}/g, " ");
+}
+export function extractSfc(src, label, kind) {
+    let html = src;
+    let title;
+    let description;
+    if (kind === "astro") {
+        const fm = /^---\r?\n[\s\S]*?\r?\n---\r?\n?/.exec(html);
+        if (fm)
+            html = html.slice(fm[0].length);
+    }
+    else if (kind === "vue") {
+        const script = [...src.matchAll(/<script\b[^>]*>([\s\S]*?)<\/script>/gi)].map((m) => m[1]).join("\n");
+        title = /\btitle\s*:\s*(["'`])(.+?)\1/.exec(script)?.[2];
+        description = /\bdescription\s*:\s*(["'`])(.+?)\1/.exec(script)?.[2];
+        const start = html.indexOf("<template");
+        const end = html.lastIndexOf("</template>");
+        if (start >= 0 && end > start)
+            html = html.slice(html.indexOf(">", start) + 1, end);
+    }
+    else {
+        html = html.replace(/<script\b[\s\S]*?<\/script>/gi, "").replace(/<style\b[\s\S]*?<\/style>/gi, "");
+    }
+    // layout props: <Layout title="…" description="…">
+    title ??= /<[A-Z][\w.]*\b[^>]*\stitle=(["'])(.*?)\1/.exec(html)?.[2];
+    description ??= /<[A-Z][\w.]*\b[^>]*\sdescription=(["'])(.*?)\1/.exec(html)?.[2];
+    html = replaceExpressions(html.replace(/<svelte:head>/g, "<head>").replace(/<\/svelte:head>/g, "</head>"));
+    const x = extractHtml(`<html><body>${html}</body></html>`, label, "file");
+    x.raw = src;
+    x.kind = "sfc";
+    if (!x.title && title)
+        x.title = title;
+    if (x.description === undefined && description)
+        x.description = description;
+    const headTitle = /<title>([^<{]+)<\/title>/i.exec(src)?.[1];
+    if (!x.title && headTitle)
+        x.title = clean(headTitle);
+    x.hasDate = x.hasDate || DATE_WORDS_RE.test(src);
+    delete x.wordsJsOff;
+    return x;
+}
+// ---------------------------------------------------------------------------
+// JSX / TSX
+const CODEY = /(;|=>|===|!==|&&|\|\||\bconst\b|\breturn\b|\bfunction\b|\bimport\b|\bexport\b|\(\)|className|onClick)/;
+function looksLikeCopy(s) {
+    const t = s.trim();
+    if (t.length < 12 || !/\p{L}/u.test(t))
+        return false;
+    if (/^(https?:|\/|\.\/|\.\.\/|@|#)/.test(t) || /\.(png|jpe?g|svg|webp|css|js|ts|tsx)$/i.test(t))
+        return false;
+    const words = t.split(/\s+/);
+    if (words.length < 3)
+        return false;
+    const classy = words.filter((w) => /[-:[\]]/.test(w) || /^\d/.test(w)).length;
+    if (classy / words.length > 0.4)
+        return false;
+    return /^[\p{Lu}"“]/u.test(t) || /[.?!:]$/.test(t);
+}
+function stripJsxInner(s) {
+    return clean(replaceExpressions(s).replace(/<[^>]+>/g, " "));
+}
+export function extractJsx(src, label) {
+    const code = src
+        .replace(/\{\/\*[\s\S]*?\*\/\}/g, " ")
+        .replace(/\/\*[\s\S]*?\*\//g, " ")
+        .replace(/^\s*\/\/.*$/gm, " ")
+        .replace(/^\s*import\s[\s\S]*?from\s+["'][^"']+["'];?\s*$/gm, " ")
+        .replace(/^\s*import\s+["'][^"']+["'];?\s*$/gm, " ");
+    const title = /\btitle\s*:\s*(["'`])((?:(?!\1).)+)\1/.exec(code)?.[2] ?? (/<title>([^<{]+)<\/title>/i.exec(code)?.[1]?.trim() || undefined);
+    const description = /\bdescription\s*:\s*(["'`])((?:(?!\1).)+)\1/.exec(code)?.[2] ??
+        /<meta\s+name=["']description["']\s+content=["']([^"']+)["']/i.exec(code)?.[1];
+    const headings = [...code.matchAll(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi)].map((m) => ({ level: Number(m[1]), text: stripJsxInner(m[2]), index: m.index ?? 0 }));
+    const firstH1 = headings.find((h) => h.level === 1);
+    const pieces = [];
+    for (const m of code.matchAll(/<(p|li|dd|dt|summary|blockquote|figcaption)\b[^>]*>([\s\S]*?)<\/\1>/gi)) {
+        const t = stripJsxInner(m[2]);
+        if (t && !CODEY.test(t))
+            pieces.push({ index: m.index ?? 0, text: t });
+    }
+    // copy kept in data (arrays/objects) is common in vibe-coded pages
+    for (const m of code.matchAll(/(["'`])((?:\\.|(?!\1)[^\\\n])*)\1/g)) {
+        const t = m[2];
+        if (m[1] === "`" && t.includes("${"))
+            continue;
+        if (looksLikeCopy(t) && !pieces.some((p) => p.text.includes(t)))
+            pieces.push({ index: m.index ?? 0, text: clean(t) });
+    }
+    // stray JSX text nodes
+    for (const m of code.matchAll(/>([^<>{}]+)</g)) {
+        const t = clean(m[1]);
+        if (t.split(" ").length >= 4 && !CODEY.test(t) && !pieces.some((p) => p.text.includes(t)))
+            pieces.push({ index: m.index ?? 0, text: t });
+    }
+    pieces.sort((a, b) => a.index - b.index);
+    const blocks = pieces.map((p) => p.text);
+    const afterH1 = firstH1 ? pieces.filter((p) => p.index < firstH1.index).length : 0;
+    const links = [...code.matchAll(/<(?:a|Link|NavLink|NuxtLink)\b[^>]*?\s(?:href|to)\s*=\s*\{?\s*(["'`])([^"'`]+)\1/g)].map((m) => m[2]);
+    const faqQuestions = new Set();
+    for (const h of headings)
+        if (questionish(h.text))
+            faqQuestions.add(h.text);
+    for (const m of code.matchAll(/\b(?:q|question|name|title)\s*:\s*(["'`])((?:(?!\1).)+\?)\1/g))
+        faqQuestions.add(clean(m[2]));
+    for (const b of blocks)
+        if (questionish(b))
+            faqQuestions.add(b);
+    return {
+        source: "file",
+        kind: "jsx",
+        label,
+        raw: src,
+        ...(title ? { title } : {}),
+        ...(description ? { description } : {}),
+        h1: headings.filter((h) => h.level === 1).map((h) => h.text),
+        headings: headings.map(({ level, text }) => ({ level, text })),
+        blocks,
+        afterH1,
+        text: blocks.join("\n"),
+        links,
+        faqQuestions: [...faqQuestions],
+        jsonLd: [],
+        jsonLdErrors: 0,
+        hasDate: DATE_WORDS_RE.test(code),
+        notes: ["JSX/TSX text is extracted roughly (headings, paragraphs, copy in string literals). For an exact check, lint the built page with url."],
+    };
+}
+export function extractFile(src, label, ext) {
+    switch (ext) {
+        case ".md":
+        case ".mdx":
+        case ".markdown":
+        case ".txt":
+            return extractMarkdown(src, label);
+        case ".html":
+        case ".htm":
+            return extractHtml(src, label);
+        case ".astro":
+            return extractSfc(src, label, "astro");
+        case ".vue":
+            return extractSfc(src, label, "vue");
+        case ".svelte":
+            return extractSfc(src, label, "svelte");
+        case ".tsx":
+        case ".jsx":
+        case ".ts":
+        case ".js":
+            return extractJsx(src, label);
+        default:
+            return /<html|<body|<p[\s>]/i.test(src) ? extractHtml(src, label) : extractMarkdown(src, label);
+    }
+}
+/** The first 1–3 sentences after the H1. */
+export function firstSentences(x, max = 3) {
+    const out = [];
+    const blocks = x.blocks.slice(x.afterH1).length ? x.blocks.slice(x.afterH1) : x.blocks;
+    for (const b of blocks) {
+        if (countWords(b) < 4 && !out.length)
+            continue; // skip tiny eyebrow/labels before the lead
+        for (const s of splitSentences(b)) {
+            out.push(s);
+            if (out.length >= max)
+                return out;
+        }
+        if (out.length)
+            break; // the first paragraph only
+    }
+    return out;
+}
+export function leadText(x, chars = 200) {
+    const blocks = x.blocks.slice(x.afterH1).length ? x.blocks.slice(x.afterH1) : x.blocks;
+    return blocks.join(" ").slice(0, chars);
+}