katman-mcp 0.1.0

package/dist/tools/method/core/research.js

@@ -0,0 +1,448 @@
+import { domainOf, mdTable, normalizeSiteUrl } from "../../../util.js";
+import { FILL, brandSentence, containsPhrase, hasNotX, hasPlaceholder, isFilled, isoDay, normKey } from "../lib/text.js";
+import { runDate } from "../lib/state.js";
+export function emptyResearch(siteUrl, languages = []) {
+    let url = "";
+    let domain = "";
+    if (siteUrl) {
+        try {
+            url = normalizeSiteUrl(siteUrl);
+            domain = domainOf(url);
+        }
+        catch {
+            // leave empty
+        }
+    }
+    return {
+        version: 1,
+        updatedAt: new Date().toISOString(),
+        site: { url, domain, languages },
+        brand: { name: "", category: "" },
+        situations: [],
+        competitors: [],
+        proof: [],
+        keywords: [],
+        questions: [],
+    };
+}
+/** research.json skeleton with [FILL] markers (book ch. 7 template). */
+export function templateResearch(siteUrl, languages = []) {
+    const r = emptyResearch(siteUrl, languages);
+    if (!r.site.url) {
+        r.site.url = FILL;
+        r.site.domain = FILL;
+    }
+    if (!r.site.languages.length)
+        r.site.languages = ["en"];
+    r.brand = {
+        name: FILL,
+        alternateNames: [],
+        category: FILL,
+        notX: FILL,
+        oneLiner: { brings: FILL, weDo: FILL, gets: FILL, fear: FILL },
+        pricing: FILL,
+        company: FILL,
+        country: FILL,
+        socials: [],
+    };
+    return r;
+}
+// ---------------------------------------------------------------------------
+// merge
+const cleanStr = (s) => (typeof s === "string" && s.trim() ? s.trim() : undefined);
+function nextId(prefix, existing) {
+    let max = 0;
+    for (const id of existing) {
+        const m = new RegExp(`^${prefix}(\\d+)$`).exec(id);
+        if (m)
+            max = Math.max(max, Number(m[1]));
+    }
+    return `${prefix}${max + 1}`;
+}
+function unionStrings(a = [], b = []) {
+    const out = [...a];
+    for (const x of b)
+        if (x && !out.some((y) => normKey(y) === normKey(x)))
+            out.push(x);
+    return out;
+}
+function normUrlPath(u) {
+    let s = u.trim();
+    if (/^https?:\/\//i.test(s)) {
+        try {
+            s = new URL(s).pathname;
+        }
+        catch {
+            // keep
+        }
+    }
+    if (!s.startsWith("/"))
+        s = "/" + s;
+    return s;
+}
+/** Merge `input` into `base`. Lists merge by id / name / sentence unless replaceLists (then given lists replace). */
+export function mergeResearch(base, input, replaceLists = false) {
+    const r = JSON.parse(JSON.stringify(base));
+    const changes = [];
+    if (input.site) {
+        if (cleanStr(input.site.url)) {
+            try {
+                const url = normalizeSiteUrl(input.site.url);
+                r.site.url = url;
+                r.site.domain = domainOf(url);
+                changes.push(`site.url = ${url}`);
+            }
+            catch {
+                r.site.url = input.site.url.trim();
+                changes.push(`site.url = ${r.site.url} (not a valid URL)`);
+            }
+        }
+        if (Array.isArray(input.site.languages) && input.site.languages.length) {
+            r.site.languages = [...new Set(input.site.languages.map((l) => l.trim().toLowerCase()).filter(Boolean))];
+            changes.push(`site.languages = ${r.site.languages.join(", ")}`);
+        }
+        if (Array.isArray(input.site.previewUrls)) {
+            r.site.previewUrls = replaceLists ? input.site.previewUrls : unionStrings(r.site.previewUrls, input.site.previewUrls);
+            changes.push("site.previewUrls updated");
+        }
+    }
+    if (input.brand) {
+        const b = input.brand;
+        const scalar = ["name", "category", "notX", "sentence", "pricing", "company", "country"];
+        for (const k of scalar) {
+            const v = cleanStr(b[k]);
+            if (v !== undefined) {
+                r.brand[k] = v;
+                changes.push(`brand.${k}`);
+            }
+        }
+        if (b.oneLiner) {
+            const cur = { brings: "", weDo: "", gets: "", fear: "", ...(r.brand.oneLiner ?? {}) };
+            for (const k of ["brings", "weDo", "gets", "fear"]) {
+                const v = cleanStr(b.oneLiner[k]);
+                if (v !== undefined) {
+                    cur[k] = v;
+                    changes.push(`brand.oneLiner.${k}`);
+                }
+            }
+            r.brand.oneLiner = cur;
+        }
+        if (Array.isArray(b.alternateNames)) {
+            r.brand.alternateNames = replaceLists ? b.alternateNames.filter(Boolean) : unionStrings(r.brand.alternateNames, b.alternateNames);
+            changes.push("brand.alternateNames");
+        }
+        if (Array.isArray(b.socials)) {
+            r.brand.socials = replaceLists ? b.socials.filter(Boolean) : unionStrings(r.brand.socials, b.socials);
+            changes.push("brand.socials");
+        }
+    }
+    if (input.situations) {
+        const list = replaceLists ? [] : r.situations.map((s) => ({ ...s }));
+        let added = 0;
+        let updated = 0;
+        for (const s of input.situations) {
+            const sentence = cleanStr(s.sentence);
+            if (!sentence)
+                continue;
+            const found = (s.id ? list.find((x) => x.id === s.id) : undefined) ?? list.find((x) => normKey(x.sentence) === normKey(sentence));
+            if (found) {
+                found.sentence = sentence;
+                if (cleanStr(s.url))
+                    found.url = normUrlPath(s.url);
+                if (s.status)
+                    found.status = s.status;
+                updated++;
+            }
+            else {
+                const id = cleanStr(s.id) && !list.some((x) => x.id === s.id) ? s.id : nextId("s", list.map((x) => x.id));
+                list.push({ id, sentence, ...(cleanStr(s.url) ? { url: normUrlPath(s.url) } : {}), status: s.status ?? "planned" });
+                added++;
+            }
+        }
+        // ids for any legacy entries without one
+        for (const s of list)
+            if (!s.id)
+                s.id = nextId("s", list.map((x) => x.id));
+        r.situations = list;
+        changes.push(`situations: ${added} added, ${updated} updated${replaceLists ? " (list replaced)" : ""}`);
+    }
+    if (input.competitors) {
+        const list = replaceLists ? [] : r.competitors.map((c) => ({ ...c }));
+        let added = 0;
+        let updated = 0;
+        for (const c of input.competitors) {
+            const name = cleanStr(c.name);
+            if (!name)
+                continue;
+            const found = list.find((x) => normKey(x.name) === normKey(name));
+            const fields = {};
+            for (const k of ["url", "what", "price", "forWhom", "source", "checkedAt"]) {
+                const v = cleanStr(c[k]);
+                if (v !== undefined)
+                    fields[k] = v;
+            }
+            if (found) {
+                Object.assign(found, fields, { name });
+                updated++;
+            }
+            else {
+                list.push({ name, url: "", ...fields });
+                added++;
+            }
+        }
+        r.competitors = list;
+        changes.push(`competitors: ${added} added, ${updated} updated${replaceLists ? " (list replaced)" : ""}`);
+    }
+    if (input.proof) {
+        const list = replaceLists ? [] : r.proof.map((p) => ({ ...p }));
+        let n = 0;
+        for (const p of input.proof) {
+            const claim = cleanStr(p.claim);
+            if (!claim)
+                continue;
+            const found = list.find((x) => normKey(x.claim) === normKey(claim));
+            const fields = {};
+            for (const k of ["value", "source", "date"]) {
+                const v = cleanStr(p[k]);
+                if (v !== undefined)
+                    fields[k] = v;
+            }
+            if (found)
+                Object.assign(found, fields);
+            else
+                list.push({ claim, source: "", ...fields });
+            n++;
+        }
+        r.proof = list;
+        changes.push(`proof: ${n} merged${replaceLists ? " (list replaced)" : ""}`);
+    }
+    if (input.keywords) {
+        const list = replaceLists ? [] : r.keywords.map((k) => ({ ...k }));
+        let n = 0;
+        for (const k of input.keywords) {
+            const term = cleanStr(k.term);
+            if (!term)
+                continue;
+            const found = list.find((x) => normKey(x.term) === normKey(term));
+            if (found) {
+                if (cleanStr(k.page))
+                    found.page = normUrlPath(k.page);
+                if (k.source)
+                    found.source = k.source;
+            }
+            else
+                list.push({ term, ...(cleanStr(k.page) ? { page: normUrlPath(k.page) } : {}), ...(k.source ? { source: k.source } : {}) });
+            n++;
+        }
+        r.keywords = list;
+        changes.push(`keywords: ${n} merged${replaceLists ? " (list replaced)" : ""}`);
+    }
+    if (input.questions) {
+        r.questions = mergeQuestions(replaceLists ? [] : r.questions, input.questions);
+        changes.push(`questions: now ${r.questions.length}${replaceLists ? " (list replaced)" : ""}`);
+    }
+    return { research: r, changes };
+}
+export function mergeQuestions(base, incoming = []) {
+    const list = base.map((q) => ({ ...q }));
+    for (const q of incoming) {
+        const text = cleanStr(q.text);
+        if (!text)
+            continue;
+        const found = (q.id ? list.find((x) => x.id === q.id) : undefined) ?? list.find((x) => normKey(x.text) === normKey(text));
+        const type = q.type ?? found?.type ?? "situation";
+        if (found) {
+            found.text = text;
+            found.type = type;
+            if (q.lang)
+                found.lang = q.lang;
+        }
+        else {
+            const prefix = type === "diagnostic" ? "t" : "q";
+            const id = cleanStr(q.id) && !list.some((x) => x.id === q.id) ? q.id : nextId(prefix, list.map((x) => x.id));
+            list.push({ id, type, text, ...(q.lang ? { lang: q.lang } : {}) });
+        }
+    }
+    return list;
+}
+// ---------------------------------------------------------------------------
+// validation
+function placeholderPaths(x, path = "") {
+    if (typeof x === "string")
+        return hasPlaceholder(x) ? [path] : [];
+    if (Array.isArray(x))
+        return x.flatMap((v, i) => placeholderPaths(v, `${path}[${i}]`));
+    if (x && typeof x === "object")
+        return Object.entries(x).flatMap(([k, v]) => placeholderPaths(v, path ? `${path}.${k}` : k));
+    return [];
+}
+const DATE_RE = /^\d{4}-\d{2}(-\d{2})?$/;
+const SLUG_RE = /^\/[a-z0-9\-/]*$/;
+export function brandNames(r) {
+    const names = [r.brand.name, ...(r.brand.alternateNames ?? [])].filter(isFilled).map((s) => s.trim());
+    // also the bare first word of a multi-word brand when it's distinctive (≥4 letters)
+    return [...new Set(names)];
+}
+/** Warnings the agent must fix (book rules: unbranded situations, sourced competitors and proof, no [FILL]). */
+export function validateResearch(r) {
+    const w = [];
+    const names = brandNames(r);
+    if (!isFilled(r.site.url))
+        w.push("site.url is missing: pass site_url to katman_start or data.site.url.");
+    if (!isFilled(r.brand.name))
+        w.push("brand.name is missing: one distinctive name (the book used a two-word name to avoid confusion).");
+    if (!isFilled(r.brand.category))
+        w.push("brand.category is missing (e.g. \"AI invoice scanner\"): assistants answer \"best <category>\" questions with it.");
+    const ol = r.brand.oneLiner;
+    const missingParts = ["brings", "weDo", "gets", "fear"].filter((k) => !isFilled(ol?.[k]));
+    if (missingParts.length)
+        w.push(`One sentence: missing part(s) ${missingParts.join(", ")} (brings → weDo → gets → fear is unnecessary).`);
+    if (!isFilled(r.brand.notX))
+        w.push("brand.notX is missing: ask ChatGPT logged out \"What is <brand>?\"; write what it gets confused with, or \"none\".");
+    if (!isFilled(r.brand.pricing))
+        w.push("brand.pricing is missing: copy it exactly from the pricing page (it feeds llms.txt and schema).");
+    for (const s of r.situations) {
+        for (const n of names) {
+            if (containsPhrase(s.sentence, n))
+                w.push(`Situation ${s.id} contains the brand name "${n}": situations must be the customer's unbranded sentence.`);
+        }
+        if (s.url && !SLUG_RE.test(s.url))
+            w.push(`Situation ${s.id} URL "${s.url}" isn't lowercase ASCII with hyphens.`);
+    }
+    if (r.situations.length < 10)
+        w.push(`${r.situations.length} situation(s): the book asks for 10–12 unbranded customer sentences.`);
+    for (const c of r.competitors) {
+        if (!isFilled(c.url))
+            w.push(`Competitor "${c.name}" has no url: take its facts from its own site.`);
+        if (!isFilled(c.checkedAt))
+            w.push(`Competitor "${c.name}" has no checkedAt date (YYYY-MM-DD).`);
+        else if (!DATE_RE.test(c.checkedAt))
+            w.push(`Competitor "${c.name}" checkedAt "${c.checkedAt}" isn't YYYY-MM-DD.`);
+    }
+    if (r.competitors.length < 3)
+        w.push(`${r.competitors.length} competitor(s): list at least 3 the assistants name, each with url + checkedAt.`);
+    for (const p of r.proof) {
+        if (!isFilled(p.source))
+            w.push(`Proof "${p.claim}" has no source: only verifiable numbers go on the site.`);
+        if (!isFilled(p.date))
+            w.push(`Proof "${p.claim}" has no date.`);
+    }
+    for (const q of r.questions) {
+        if (q.type !== "diagnostic" && names.some((n) => containsPhrase(q.text, n))) {
+            w.push(`Question ${q.id} mentions the brand: only the 2 diagnostic questions may be branded (branded questions inflate visibility).`);
+        }
+    }
+    const leftovers = placeholderPaths(r).filter((p) => !p.startsWith("questions"));
+    if (leftovers.length)
+        w.push(`[FILL]/[DOLDUR] left in: ${leftovers.slice(0, 12).join(", ")}${leftovers.length > 12 ? ` (+${leftovers.length - 12})` : ""}.`);
+    return w;
+}
+export function isSourcedCompetitor(c) {
+    return isFilled(c.url) && isFilled(c.checkedAt);
+}
+export function researchCompleteness(r) {
+    if (!r)
+        return { percent: 0, items: [], gaps: ["No research yet."], blocking: ["research"] };
+    const ol = r.brand.oneLiner;
+    const parts = ["brings", "weDo", "gets", "fear"].filter((k) => isFilled(ol?.[k])).length;
+    const sourced = r.competitors.filter(isSourcedCompetitor).length;
+    const primary = (r.site.languages[0] ?? "en").toLowerCase();
+    const primaryQs = r.questions.filter((q) => !q.lang || q.lang.toLowerCase() === primary);
+    const nonDiag = primaryQs.filter((q) => q.type !== "diagnostic").length;
+    const diag = primaryQs.filter((q) => q.type === "diagnostic").length;
+    const items = [
+        { key: "name", label: "brand name", score: isFilled(r.brand.name) ? 1 : 0, detail: isFilled(r.brand.name) ? r.brand.name : "missing" },
+        { key: "category", label: "category", score: isFilled(r.brand.category) ? 1 : 0, detail: isFilled(r.brand.category) ? r.brand.category : "missing" },
+        { key: "oneLiner", label: "one sentence (4 parts)", score: parts / 4, detail: `${parts}/4 parts` },
+        { key: "notX", label: "not-X note", score: isFilled(r.brand.notX) ? 1 : 0, detail: isFilled(r.brand.notX) ? (hasNotX(r.brand.notX) ? "set" : "none (checked)") : "missing" },
+        { key: "situations", label: "≥10 situations", score: Math.min(1, r.situations.length / 10), detail: `${r.situations.length}/10` },
+        { key: "competitors", label: "≥3 competitors with url + date", score: Math.min(1, sourced / 3), detail: `${sourced}/3 sourced (${r.competitors.length} listed)` },
+        { key: "questions", label: "question set (12 + 2)", score: Math.min(1, (Math.min(nonDiag, 12) + Math.min(diag, 2)) / 14), detail: `${nonDiag} + ${diag}` },
+    ];
+    const percent = Math.round((items.reduce((a, i) => a + i.score, 0) / items.length) * 100);
+    const gaps = items.filter((i) => i.score < 1).map((i) => `${i.label}: ${i.detail}`);
+    if (!isFilled(r.site.url))
+        gaps.push("site URL: missing");
+    if (!isFilled(r.brand.pricing))
+        gaps.push("pricing: missing");
+    if (!r.proof.length)
+        gaps.push("proof: none yet (optional, but only sourced numbers may appear on pages)");
+    const blocking = [];
+    if (!isFilled(r.brand.name))
+        blocking.push("brand name");
+    if (!isFilled(r.brand.category))
+        blocking.push("category");
+    if (parts < 4)
+        blocking.push("one sentence (4 parts)");
+    if (r.situations.length < 3)
+        blocking.push("at least 3 situations");
+    if (!r.questions.length)
+        blocking.push("question set");
+    return { percent, items, gaps, blocking };
+}
+// ---------------------------------------------------------------------------
+// research.md
+function v(x, fill = FILL) {
+    return isFilled(x) ? String(x).trim() : fill;
+}
+export function renderResearchMd(r, runs, lang = "en") {
+    const L = (en, tr) => (lang === "tr" ? tr : en);
+    const fill = lang === "tr" ? "[DOLDUR]" : FILL;
+    const out = [];
+    out.push(`# research.md`);
+    out.push(L(`_Generated by Katman from .katman/research.json on ${isoDay(new Date())}. Don't edit this file: change research through \`katman_research\` (mode "update") — this file is rewritten._`, `_Katman tarafından .katman/research.json’dan ${isoDay(new Date())} tarihinde üretildi. Bu dosyayı düzenlemeyin: araştırmayı \`katman_research\` (mode "update") ile değiştirin — bu dosya yeniden yazılır._`));
+    out.push(`## ${L("Brand", "Marka")}`);
+    const sentence = brandSentence(r);
+    const ol = r.brand.oneLiner;
+    out.push([
+        `- ${L("Name", "Ad")}: ${v(r.brand.name, fill)}${r.brand.alternateNames?.length ? ` (${L("also", "diğer adlar")}: ${r.brand.alternateNames.join(", ")})` : ""}`,
+        `- ${L("Category", "Kategori")}: ${v(r.brand.category, fill)}`,
+        `- ${L("Is not", "Değildir")}: ${v(r.brand.notX, fill)}`,
+        `- ${L("One sentence", "Tek cümle")}: ${sentence ?? fill}`,
+        `  - ${L("brings", "getirir")}: ${v(ol?.brings, fill)}`,
+        `  - ${L("we do", "yaparız")}: ${v(ol?.weDo, fill)}`,
+        `  - ${L("they get", "eline geçer")}: ${v(ol?.gets, fill)}`,
+        `  - ${L("fear is unnecessary", "korku gereksiz")}: ${v(ol?.fear, fill)}`,
+        `- ${L("Pricing (exactly as on the site)", "Fiyat (sitedeki ile aynı)")}: ${v(r.brand.pricing, fill)}`,
+        `- ${L("Company", "Şirket")}: ${v(r.brand.company, fill)}${isFilled(r.brand.country) ? ` (${r.brand.country})` : ""}`,
+        `- ${L("Site", "Site")}: ${v(r.site.url, fill)} · ${L("languages", "diller")}: ${r.site.languages.join(", ") || fill}`,
+        ...(r.brand.socials?.length ? [`- ${L("Social accounts", "Sosyal hesaplar")}: ${r.brand.socials.join(", ")}`] : []),
+        ...(r.site.previewUrls?.length ? [`- ${L("Preview URLs (must redirect or be noindex)", "Önizleme adresleri (yönlenmeli ya da noindex olmalı)")}: ${r.site.previewUrls.join(", ")}`] : []),
+    ].join("\n"));
+    out.push(`## ${L("Customer situations (10–12, unbranded)", "Müşteri durumları (10-12, markasız)")}`);
+    out.push(r.situations.length
+        ? r.situations.map((s, i) => `${i + 1}. "${s.sentence}"   → ${L("page", "sayfa")}: ${s.url ?? fill} (${s.status ?? "planned"}) \`${s.id}\``).join("\n")
+        : `1. "${L("[the sentence your customer would type into ChatGPT]", "[müşterinin ChatGPT'ye yazacağı cümle]")}"   → ${L("page", "sayfa")}: /[url]`);
+    const last = runs[runs.length - 1];
+    out.push(`## ${L("Assistant measurement", "Asistan ölçümü")}${last ? ` (${isoDay(runDate(last) ?? last.ts)}, ${last.answers.some((a) => a.source === "manual") ? L("includes manual checks", "elle kontroller dahil") : "API"})` : ""}`);
+    if (last && last.answers.length) {
+        const rows = last.answers.slice(0, 60).map((a) => {
+            const own = a.citations.find((c) => r.site.domain && c.domain.endsWith(r.site.domain));
+            return [
+                a.question,
+                a.family || a.model,
+                a.error ? `error: ${a.error}` : a.mentioned ? L(`yes${a.rank ? ` (#${a.rank})` : ""}`, `evet${a.rank ? ` (#${a.rank})` : ""}`) : L("no", "hayır"),
+                a.namedProducts.slice(0, 3).join(", "),
+                own ? own.url : a.citations[0]?.url ?? "",
+            ];
+        });
+        out.push(mdTable([L("question", "soru"), L("assistant", "asistan"), L("mentioned us?", "bizi andı mı"), L("whom it recommended", "kimi önerdi"), L("page it cited", "kaynak gösterdiği sayfa")], rows));
+        out.push(L(`Rates: mentioned ${Math.round(last.summary.mentionRate * 100)}% · recommended first ${Math.round(last.summary.recommendedFirstRate * 100)}% · cited ${Math.round(last.summary.citationRate * 100)}% (${last.summary.answered} answers; one run is a low sample).`, `Oranlar: anılma %${Math.round(last.summary.mentionRate * 100)} · ilk öneri %${Math.round(last.summary.recommendedFirstRate * 100)} · kaynak gösterilme %${Math.round(last.summary.citationRate * 100)} (${last.summary.answered} cevap; tek ölçüm düşük örnektir).`));
+    }
+    else {
+        out.push(`| ${L("question | assistant | mentioned us? | whom it recommended | page it cited", "soru | asistan | bizi andı mı | kimi önerdi | kaynak gösterdiği sayfa")} |\n| --- | --- | --- | --- | --- |\n\n${L("No measurement yet: katman_visibility_run (dry_run first) or katman_manual_check.", "Henüz ölçüm yok: katman_visibility_run (önce dry_run) ya da katman_manual_check.")}`);
+    }
+    out.push(`## ${L("Competitors (each from its own site, with the date checked)", "Rakipler (her biri kendi sitesinden, kontrol tarihiyle)")}`);
+    out.push(mdTable([L("name", "ad"), L("what it does", "ne yapar"), L("price", "fiyat"), L("for whom", "kime uygun"), L("source", "kaynak"), L("date", "tarih")], r.competitors.length
+        ? r.competitors.map((c) => [c.name, c.what ?? "", c.price ?? "", c.forWhom ?? "", c.source ?? c.url ?? "", c.checkedAt ?? fill])
+        : [[fill, "", "", "", "", ""]]));
+    out.push(`## ${L("Proof (verifiable only)", "Kanıt (yalnızca doğrulanabilir)")}`);
+    out.push(r.proof.length ? r.proof.map((p) => `- ${p.claim}${p.value ? `: ${p.value}` : ""} — ${v(p.source, fill)} — ${v(p.date, fill)}`).join("\n") : `- [${L("number", "sayı")}] — [${L("source", "kaynak")}] — [${L("date", "tarih")}]`);
+    out.push(`## ${L("Search keywords", "Aranan kelimeler")}`);
+    out.push(r.keywords.length ? r.keywords.map((k) => `- ${k.term} → ${k.page ?? fill}${k.source ? ` (${k.source})` : ""}`).join("\n") : `- [${L("keyword", "kelime")}] → [${L("which page", "hangi sayfa")}]`);
+    out.push(`## ${L("Question set (12 unbranded + 2 diagnostic)", "Soru seti (12 markasız + 2 teşhis)")}`);
+    out.push(r.questions.length
+        ? r.questions.map((q) => `- \`${q.id}\` [${q.type}${q.lang ? `, ${q.lang}` : ""}] ${q.text}`).join("\n")
+        : L("No questions yet: katman_research {\"generate_questions\": true} once there are ≥3 situations.", "Henüz soru yok: en az 3 durum olunca katman_research {\"generate_questions\": true}."));
+    return out.join("\n\n") + "\n";
+}