katman-mcp 0.1.0

package/dist/tools/crawl/keywords.js

@@ -0,0 +1,178 @@
+import { httpGet, sleep as realSleep } from "../../net.js";
+import { mdTable } from "../../util.js";
+import { collapse, fold } from "./common/text.js";
+export const MAX_REQUESTS = 40;
+export const DELAY_MS = 350;
+const LETTERS = {
+    tr: "abcçdefgğhıijklmnoöprsştuüvyz",
+    de: "abcdefghijklmnopqrstuvwxyzäöü",
+    es: "abcdefghijklmnñopqrstuvwxyz",
+};
+export function suggestUrl(q, lang, country) {
+    return `https://suggestqueries.google.com/complete/search?client=firefox&hl=${encodeURIComponent(lang)}&gl=${encodeURIComponent(country)}&ie=utf-8&oe=utf-8&q=${encodeURIComponent(q)}`;
+}
+export function parseSuggestions(body) {
+    try {
+        const j = JSON.parse(body);
+        if (Array.isArray(j) && Array.isArray(j[1]))
+            return j[1].filter((x) => typeof x === "string");
+    }
+    catch {
+        /* not JSON */
+    }
+    return [];
+}
+export function queriesFor(seed, lang, expand) {
+    const s = collapse(seed);
+    if (!expand)
+        return [s];
+    const letters = [...(LETTERS[lang] ?? "abcdefghijklmnopqrstuvwxyz")];
+    const qs = [s, ...letters.map((l) => `${s} ${l}`), ..."0123456789".split("").map((d) => `${s} ${d}`)];
+    return qs.slice(0, MAX_REQUESTS);
+}
+const Q_EN = /^(how|what|why|can|is|are|does|do|which|where|when|should|who)\b|\b(how to|what is|why does|can i|can you)\b/;
+const Q_TR = /\b(nasil|nedir|neden|nicin|hangi|kac|ne zaman)\b|\s(mi|mu)(\s|$)/;
+const CMP_EN = /\b(best|vs|versus|alternative|alternatives|compare|comparison|top|review|reviews|like)\b/;
+const CMP_TR = /\b(en iyi|alternatif|alternatifleri|alternatifi|karsilastirma|yorum|yorumlari|benzeri)\b/;
+const SIT_EN = /\b(free|without|for|with|no|instead|from)\b/;
+const SIT_TR = /\b(ucretsiz|bedava|olmadan|icin|ile|yerine)\b/;
+// First-person phrasing is how customers describe their situation to an assistant (book ch. 7).
+const FIRST_PERSON = /\b(i|i'm|im|my|me|mine)\b|\b(istiyorum|yapamiyorum|bilmiyorum|olmuyor|calismiyor|benim|var ama)\b/;
+/** Classify a suggestion by what it adds to the seed (the seed's own words don't count). */
+export function classifyTerm(term, seed = "") {
+    const t = fold(term);
+    const seedTokens = new Set(fold(seed).split(/[^\p{L}\p{N}']+/u).filter(Boolean));
+    const extra = t.split(/[^\p{L}\p{N}']+/u).filter((w) => w && !seedTokens.has(w)).join(" ");
+    if (Q_EN.test(t) || Q_TR.test(t))
+        return "single-question";
+    if (CMP_EN.test(extra) || CMP_TR.test(extra))
+        return "comparison";
+    if (FIRST_PERSON.test(extra) || SIT_EN.test(extra) || SIT_TR.test(extra))
+        return "situation";
+    return undefined;
+}
+const PAGE = {
+    "single-question": "single-question page (answer in the first 1–3 sentences, P6)",
+    situation: "situation page (P6)",
+    comparison: "honest comparison guide (P7)",
+};
+const STOP = new Set(["the", "a", "an", "and", "or", "of", "in", "on", "for", "to", "with", "ve", "ile", "bir", "icin", "da", "de"]);
+export function cluster(seed, suggestions) {
+    const seedTokens = new Set(fold(seed).split(/[^\p{L}\p{N}]+/u).filter(Boolean));
+    const groups = new Map();
+    const label = new Map();
+    for (const s of suggestions) {
+        const words = s.toLocaleLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
+        const first = words.find((w) => !seedTokens.has(fold(w)) && !STOP.has(fold(w)));
+        let key = first ? fold(first) : "(seed wording)";
+        if (key.length > 4 && key.endsWith("s"))
+            key = key.slice(0, -1);
+        if (first && !label.has(key))
+            label.set(key, first);
+        groups.set(key, [...(groups.get(key) ?? []), s]);
+    }
+    let list = [...groups.entries()].map(([key, items]) => ({ key: label.get(key) ?? key, items })).sort((a, b) => b.items.length - a.items.length || a.key.localeCompare(b.key));
+    if (list.length > 14) {
+        const big = list.filter((c) => c.items.length > 1);
+        const small = list.filter((c) => c.items.length === 1).flatMap((c) => c.items);
+        list = small.length ? [...big, { key: "other", items: small }] : big;
+    }
+    return list;
+}
+export async function runKeywords(o) {
+    const lang = (o.lang ?? "en").toLowerCase();
+    const country = (o.country ?? "us").toLowerCase();
+    const fetcher = o.fetcher ?? httpGet;
+    const pause = o.sleep ?? realSleep;
+    const delay = o.delayMs ?? DELAY_MS;
+    const qs = queriesFor(o.seed, lang, o.expand !== false);
+    const errors = [];
+    const seen = new Map();
+    const seedKey = fold(collapse(o.seed));
+    let requests = 0;
+    for (let i = 0; i < qs.length && requests < MAX_REQUESTS; i++) {
+        if (i > 0)
+            await pause(delay);
+        requests++;
+        const res = await fetcher(suggestUrl(qs[i], lang, country), { redirect: "follow", headers: { accept: "application/json,text/javascript,*/*" }, timeoutMs: 10_000 });
+        if (res.status === 429 || res.status === 403) {
+            errors.push(`HTTP ${res.status} on "${qs[i]}": Google is rate-limiting; stopped early.`);
+            break;
+        }
+        if (res.status !== 200) {
+            errors.push(`HTTP ${res.status || res.error} on "${qs[i]}"`);
+            continue;
+        }
+        for (const s of parseSuggestions(res.body)) {
+            const c = collapse(s);
+            const k = fold(c);
+            if (!c || k === seedKey || seen.has(k))
+                continue;
+            seen.set(k, c);
+        }
+    }
+    const suggestions = [...seen.values()];
+    const candidates = [];
+    for (const s of suggestions) {
+        const kind = classifyTerm(s, o.seed);
+        if (kind)
+            candidates.push({ term: s, kind, page: PAGE[kind] });
+    }
+    return { seed: collapse(o.seed), lang, country, requests, errors, suggestions, clusters: cluster(o.seed, suggestions), candidates };
+}
+/** Terms worth saving: every candidate, then the top of each cluster (max 60). */
+export function termsToSave(r) {
+    const out = new Map();
+    for (const c of r.candidates)
+        out.set(fold(c.term), c.term);
+    for (const c of r.clusters)
+        for (const t of c.items.slice(0, 3))
+            out.set(fold(t), t);
+    return [...out.values()].slice(0, 60);
+}
+/** Merge into research.keywords (source "autocomplete"); returns the number added. */
+export function mergeKeywords(research, terms) {
+    const have = new Set((research.keywords ?? []).map((k) => fold(k.term)));
+    let added = 0;
+    research.keywords ??= [];
+    for (const t of terms) {
+        if (have.has(fold(t)))
+            continue;
+        const k = { term: t, source: "autocomplete" };
+        research.keywords.push(k);
+        have.add(fold(t));
+        added++;
+    }
+    return added;
+}
+export function keywordsMarkdown(r, saved) {
+    const L = [];
+    L.push(`# Google autocomplete — "${r.seed}" (${r.lang}-${r.country.toUpperCase()})`);
+    L.push(`${r.suggestions.length} unique suggestions from ${r.requests} requests (seed + letters/digits, ${DELAY_MS} ms apart; suggestqueries.google.com only).`);
+    for (const e of r.errors.slice(0, 5))
+        L.push(`- ${e}`);
+    L.push("");
+    if (!r.suggestions.length) {
+        L.push("No suggestions: try a shorter seed in the customer's own words, or another language/country.");
+        return L.join("\n");
+    }
+    const byKind = (k) => r.candidates.filter((c) => c.kind === k);
+    for (const [kind, title] of [["single-question", "Single-question page candidates"], ["situation", "Situation page candidates"], ["comparison", "Comparison / alternatives candidates"]]) {
+        const list = byKind(kind);
+        if (!list.length)
+            continue;
+        L.push(`## ${title} (${list.length}) → ${PAGE[kind]}`);
+        for (const c of list.slice(0, 20))
+            L.push(`- ${c.term}`);
+        if (list.length > 20)
+            L.push(`- …${list.length - 20} more`);
+        L.push("");
+    }
+    L.push("## Clusters");
+    L.push(mdTable(["Cluster", "Count", "Examples"], r.clusters.slice(0, 20).map((c) => [c.key, c.items.length, c.items.slice(0, 4).join(" · ")])));
+    L.push("");
+    L.push("Autocomplete shows what people type into Google. People describe situations to ChatGPT in longer sentences: combine these with the situations in research.json (unbranded, in the customer's own words).");
+    if (saved)
+        L.push(`\nSaved: ${saved.added} new keyword(s) merged into .katman/research.json (source: autocomplete).${saved.note ? ` ${saved.note}` : ""}`);
+    return L.join("\n");
+}