mcp-astgl-knowledge 1.0.2 → 1.1.0

package/dist/related-articles.js

@@ -0,0 +1,217 @@
+#!/usr/bin/env tsx
+/**
+ * Internal linking automation via vector similarity.
+ *
+ * WHAT: Computes pairwise article similarity and injects related article links
+ * WHY: Cross-referencing boosts SEO, AI discoverability, and reader engagement
+ *
+ * Usage:
+ *   npm run related                        Compute + print JSON map
+ *   npm run related -- --inject            Also inject into Astro markdown frontmatter
+ *   npm run related -- --top 3             Number of related articles per article (default: 3)
+ *
+ * Requires: Ollama running with nomic-embed-text
+ */
+import { join } from "path";
+import { existsSync, readFileSync, writeFileSync, readdirSync } from "fs";
+import Database from "better-sqlite3";
+const DATA_DIR = join(import.meta.dirname, "..", "data");
+const KNOWLEDGE_PATH = join(DATA_DIR, "knowledge.db");
+const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434";
+const EMBED_MODEL = process.env.EMBED_MODEL || "nomic-embed-text";
+const ASTRO_ANSWERS_DIR = process.env.ASTRO_ANSWERS_DIR ||
+    join(import.meta.dirname, "..", "..", "astgl-site", "src", "content", "answers");
+const BASE_URL = "https://astgl.ai/answers";
+// --- Embedding ---
+async function embedTexts(texts) {
+    // WHAT: Batch embed via Ollama /api/embed (single call, multiple inputs)
+    // WHY: 1 API call for all articles is faster than 20 separate calls
+    const resp = await fetch(`${OLLAMA_URL}/api/embed`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ model: EMBED_MODEL, input: texts }),
+    });
+    if (!resp.ok) {
+        throw new Error(`Ollama embed failed: ${resp.status} ${await resp.text()}`);
+    }
+    const data = (await resp.json());
+    return data.embeddings;
+}
+// WHAT: Cosine similarity between two vectors
+// WHY: Standard similarity metric for embedding comparison (same as sqlite-vec uses)
+function cosineSimilarity(a, b) {
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+// --- Knowledge DB: Store related articles ---
+function storeRelatedInDb(db, relatedMap) {
+    // WHAT: Create table + upsert related article links
+    // WHY: MCP server can serve related articles alongside search results
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS article_related (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      article_url TEXT NOT NULL,
+      related_url TEXT NOT NULL,
+      related_title TEXT NOT NULL,
+      similarity_score REAL NOT NULL,
+      rank INTEGER NOT NULL,
+      UNIQUE(article_url, related_url)
+    )
+  `);
+    db.exec("CREATE INDEX IF NOT EXISTS idx_related_article ON article_related(article_url)");
+    const upsert = db.prepare(`INSERT OR REPLACE INTO article_related (article_url, related_url, related_title, similarity_score, rank)
+     VALUES (?, ?, ?, ?, ?)`);
+    const upsertAll = db.transaction(() => {
+        // Clear existing data
+        db.prepare("DELETE FROM article_related").run();
+        for (const [slug, entry] of Object.entries(relatedMap)) {
+            const articleUrl = entry.url;
+            for (let i = 0; i < entry.related.length; i++) {
+                const rel = entry.related[i];
+                upsert.run(articleUrl, `${BASE_URL}/${rel.slug}`, rel.title, rel.score, i + 1);
+            }
+        }
+    });
+    upsertAll();
+}
+// --- Astro Frontmatter Injection ---
+// WHAT: Parse YAML frontmatter from a markdown file
+// WHY: Need to read existing frontmatter, add/update `related` field, write back
+function injectRelatedFrontmatter(filePath, related) {
+    const content = readFileSync(filePath, "utf-8");
+    // Split frontmatter from body
+    const fmMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+    if (!fmMatch) {
+        console.error(`  Skipping ${filePath}: no frontmatter found`);
+        return false;
+    }
+    let frontmatter = fmMatch[1];
+    const body = fmMatch[2];
+    // Remove existing `related:` block if present
+    frontmatter = frontmatter.replace(/related:\n(?:- slug:[\s\S]*?(?=\n\w|\n---|\z))/g, "");
+    // Cleaner regex: remove the related block entirely
+    frontmatter = frontmatter.replace(/related:\n(?:(?:- (?:slug|title):.*\n|  .*\n)*)/g, "");
+    // Remove trailing whitespace
+    frontmatter = frontmatter.trimEnd();
+    // Build the related YAML block
+    const relatedYaml = related
+        .map((r) => `- slug: ${r.slug}\n  title: "${r.title.replace(/"/g, '\\"')}"`)
+        .join("\n");
+    frontmatter += `\nrelated:\n${relatedYaml}`;
+    const updated = `---\n${frontmatter}\n---\n${body}`;
+    writeFileSync(filePath, updated, "utf-8");
+    return true;
+}
+// --- CLI ---
+function parseArgs() {
+    const args = process.argv.slice(2);
+    return {
+        inject: args.includes("--inject"),
+        top: (() => {
+            const idx = args.indexOf("--top");
+            return idx >= 0 && args[idx + 1] ? parseInt(args[idx + 1], 10) : 3;
+        })(),
+    };
+}
+async function main() {
+    const { inject, top } = parseArgs();
+    console.error("=== Related Articles Generator ===\n");
+    if (!existsSync(KNOWLEDGE_PATH)) {
+        console.error("knowledge.db not found. Run 'npm run ingest' first.");
+        process.exit(1);
+    }
+    const db = new Database(KNOWLEDGE_PATH);
+    // WHAT: Only process astgl.ai canonical articles (not Substack mirrors)
+    // WHY: These are the articles we control and can inject links into
+    const articles = db
+        .prepare(`SELECT title, description, url, slug
+       FROM articles
+       WHERE url LIKE 'https://astgl.ai/answers/%'
+       ORDER BY rowid`)
+        .all();
+    console.error(`Found ${articles.length} astgl.ai articles to process\n`);
+    if (articles.length === 0) {
+        console.error("No articles found.");
+        db.close();
+        process.exit(0);
+    }
+    // Step 1: Embed all articles (title + description)
+    console.error("Embedding articles...");
+    const texts = articles.map((a) => `${a.title}. ${a.description}`);
+    const embeddings = await embedTexts(texts);
+    console.error(`  ${embeddings.length} embeddings generated\n`);
+    // Step 2: Compute pairwise similarity
+    console.error("Computing pairwise similarity...");
+    const relatedMap = {};
+    for (let i = 0; i < articles.length; i++) {
+        const article = articles[i];
+        const similarities = [];
+        for (let j = 0; j < articles.length; j++) {
+            if (i === j)
+                continue;
+            const score = Math.round(cosineSimilarity(embeddings[i], embeddings[j]) * 1000) / 1000;
+            similarities.push({ index: j, score });
+        }
+        // Sort by similarity descending, take top N
+        similarities.sort((a, b) => b.score - a.score);
+        const topRelated = similarities.slice(0, top).map((s) => ({
+            slug: articles[s.index].slug,
+            title: articles[s.index].title,
+            score: s.score,
+        }));
+        relatedMap[article.slug] = {
+            title: article.title,
+            url: article.url,
+            related: topRelated,
+        };
+        console.error(`  ${article.slug}: ${topRelated.map((r) => `${r.slug} (${r.score})`).join(", ")}`);
+    }
+    // Step 3: Store in knowledge.db
+    console.error("\nStoring in knowledge.db...");
+    storeRelatedInDb(db, relatedMap);
+    console.error(`  ${Object.keys(relatedMap).length * top} relationships stored`);
+    db.close();
+    // Step 4: Inject into Astro frontmatter (optional)
+    if (inject) {
+        console.error("\nInjecting into Astro frontmatter...");
+        if (!existsSync(ASTRO_ANSWERS_DIR)) {
+            console.error(`  Astro answers dir not found: ${ASTRO_ANSWERS_DIR}`);
+            console.error("  Set ASTRO_ANSWERS_DIR env var to override.");
+        }
+        else {
+            const mdFiles = readdirSync(ASTRO_ANSWERS_DIR).filter((f) => f.endsWith(".md"));
+            let injected = 0;
+            for (const file of mdFiles) {
+                const slug = file.replace(/\.md$/, "");
+                const entry = relatedMap[slug];
+                if (!entry) {
+                    console.error(`  ${slug}: not in knowledge base, skipping`);
+                    continue;
+                }
+                const filePath = join(ASTRO_ANSWERS_DIR, file);
+                if (injectRelatedFrontmatter(filePath, entry.related)) {
+                    console.error(`  ${slug}: injected ${entry.related.length} related links`);
+                    injected++;
+                }
+            }
+            console.error(`\n  ${injected} files updated`);
+        }
+    }
+    // Output JSON map to stdout
+    console.log(JSON.stringify(relatedMap, null, 2));
+    console.error("\n=== Done ===");
+}
+main()
+    .then(() => process.exit(0))
+    .catch((err) => {
+    console.error("Related articles failed:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=related-articles.js.map

package/dist/related-articles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"related-articles.js","sourceRoot":"","sources":["../src/related-articles.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAC1E,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AACzD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;AACtD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,CAAC;AACtE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,kBAAkB,CAAC;AAClE,MAAM,iBAAiB,GACrB,OAAO,CAAC,GAAG,CAAC,iBAAiB;IAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACnF,MAAM,QAAQ,GAAG,0BAA0B,CAAC;AAuB5C,oBAAoB;AACpB,KAAK,UAAU,UAAU,CAAC,KAAe;IACvC,yEAAyE;IACzE,oEAAoE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,UAAU,YAAY,EAAE;QAClD,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;KAC3D,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,MAAM,IAAI,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA+B,CAAC;IAC/D,OAAO,IAAI,CAAC,UAAU,CAAC;AACzB,CAAC;AAED,8CAA8C;AAC9C,qFAAqF;AACrF,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,+CAA+C;AAC/C,SAAS,gBAAgB,CACvB,EAAiC,EACjC,UAAsB;IAEtB,oDAAoD;IACpD,sEAAsE;IACtE,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;GAUP,CAAC,CAAC;IACH,EAAE,CAAC,IAAI,CACL,gFAAgF,CACjF,CAAC;IAEF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB;4BACwB,CACzB,CAAC;IAEF,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACpC,sBAAsB;QACtB,EAAE,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC,GAAG,EAAE,CAAC;QAEhD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YACvD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC7B,MAAM,CAAC,GAAG,CACR,UAAU,EACV,GAAG,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,EACzB,GAAG,CAAC,KAAK,EACT,GAAG,CAAC,KAAK,EACT,CAAC,GAAG,CAAC,CACN,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,SAAS,EAAE,CAAC;AACd,CAAC;AAED,sCAAsC;AACtC,oDAAoD;AACpD,iFAAiF;AACjF,SAAS,wBAAwB,CAC/B,QAAgB,EAChB,OAAuB;IAEvB,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEhD,8BAA8B;IAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACnE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,cAAc,QAAQ,wBAAwB,CAAC,CAAC;QAC9D,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAExB,8CAA8C;IAC9C,WAAW,GAAG,WAAW,CAAC,OAAO,CAC/B,iDAAiD,EACjD,EAAE,CACH,CAAC;IACF,mDAAmD;IACnD,WAAW,GAAG,WAAW,CAAC,OAAO,CAC/B,kDAAkD,EAClD,EAAE,CACH,CAAC;IACF,6BAA6B;IAC7B,WAAW,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC;IAEpC,+BAA+B;IAC/B,MAAM,WAAW,GAAG,OAAO;SACxB,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,WAAW,CAAC,CAAC,IAAI,eAAe,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAClE;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,WAAW,IAAI,eAAe,WAAW,EAAE,CAAC;IAE5C,MAAM,OAAO,GAAG,QAAQ,WAAW,UAAU,IAAI,EAAE,CAAC;IACpD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,cAAc;AACd,SAAS,SAAS;IAChB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QACjC,GAAG,EAAE,CAAC,GAAG,EAAE;YACT,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClC,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,CAAC,CAAC,EAAE;KACL,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;IAEpC,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAEtD,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,cAAc,CAAC,CAAC;IAExC,wEAAwE;IACxE,mEAAmE;IACnE,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CACN;;;sBAGgB,CACjB;SACA,GAAG,EAAmB,CAAC;IAE1B,OAAO,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,MAAM,iCAAiC,CAAC,CAAC;IAEzE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,mDAAmD;IACnD,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,MAAM,yBAAyB,CAAC,CAAC;IAE/D,sCAAsC;IACtC,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAClD,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,YAAY,GAA4C,EAAE,CAAC;QAEjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAS;YACtB,MAAM,KAAK,GACT,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YAC3E,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,4CAA4C;QAC5C,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI;YAC5B,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK;YAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC,CAAC;QAEJ,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG;YACzB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO,EAAE,UAAU;SACpB,CAAC;QAEF,OAAO,CAAC,KAAK,CACX,KAAK,OAAO,CAAC,IAAI,KAAK,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnF,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC9C,gBAAgB,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACjC,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,GAAG,uBAAuB,CAAC,CAAC;IAEhF,EAAE,CAAC,KAAK,EAAE,CAAC;IAEX,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEvD,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,kCAAkC,iBAAiB,EAAE,CAAC,CAAC;YACrE,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1D,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAClB,CAAC;YAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACvC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,mCAAmC,CAAC,CAAC;oBAC5D,SAAS;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;gBAC/C,IAAI,wBAAwB,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACtD,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,cAAc,KAAK,CAAC,OAAO,CAAC,MAAM,gBAAgB,CAAC,CAAC;oBAC3E,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,OAAO,QAAQ,gBAAgB,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEjD,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;AAClC,CAAC;AAED,IAAI,EAAE;KACH,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC3B,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACb,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/search.d.ts

@@ -2,10 +2,19 @@
  * Vector search logic against the pre-built knowledge.db.
  * At runtime, queries are embedded via Ollama and matched against stored vectors.
  *
- * WHAT: Provides search_articles, get_answer, and list_topics functions
+ * WHAT: Provides search_articles, get_answer, list_topics, and related functions
  * WHY: Separating search logic from MCP server keeps concerns clean
+ *
+ * Performance:
+ *   - LRU cache on embeddings (200 entries) avoids redundant Ollama calls
+ *   - Ollama calls have 10s timeout + 1 retry with 500ms delay
+ *   - Prepared statements cached at module level
+ *   - listTopics uses GROUP_CONCAT instead of in-memory join
  */
-import { SearchResult, AnswerResult, TopicEntry } from "./types.js";
-export declare function searchArticles(query: string, limit?: number): Promise<SearchResult[]>;
-export declare function getAnswer(question: string): Promise<AnswerResult>;
+import { SearchResult, AnswerResult, TopicEntry, TutorialResult, ComparisonResult, LatestArticle } from "./types.js";
+export declare function searchArticles(query: string, limit?: number, contentType?: string): Promise<SearchResult[]>;
+export declare function getAnswer(question: string, contentType?: string): Promise<AnswerResult>;
+export declare function getTutorial(query: string): Promise<TutorialResult>;
+export declare function compareTopics(topicA: string, topicB: string): Promise<ComparisonResult>;
+export declare function getLatest(limit?: number): LatestArticle[];
 export declare function listTopics(): TopicEntry[];

package/dist/search.js

@@ -2,8 +2,14 @@
  * Vector search logic against the pre-built knowledge.db.
  * At runtime, queries are embedded via Ollama and matched against stored vectors.
  *
- * WHAT: Provides search_articles, get_answer, and list_topics functions
+ * WHAT: Provides search_articles, get_answer, list_topics, and related functions
  * WHY: Separating search logic from MCP server keeps concerns clean
+ *
+ * Performance:
+ *   - LRU cache on embeddings (200 entries) avoids redundant Ollama calls
+ *   - Ollama calls have 10s timeout + 1 retry with 500ms delay
+ *   - Prepared statements cached at module level
+ *   - listTopics uses GROUP_CONCAT instead of in-memory join
  */
 import { join } from "path";
 import { existsSync } from "fs";
@@ -12,6 +18,7 @@ import * as sqliteVec from "sqlite-vec";
 const DB_PATH = join(import.meta.dirname, "..", "data", "knowledge.db");
 const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434";
 const EMBED_MODEL = process.env.EMBED_MODEL || "nomic-embed-text";
+// --- Database Connection (lazy singleton) ---
 let db = null;
 function getDb() {
     if (!db) {
@@ -23,21 +30,100 @@ function getDb() {
     }
     return db;
 }
+// --- Embedding Cache (LRU) ---
+// WHAT: In-memory cache for embedding vectors, keyed by normalized query text
+// WHY: Same query → same embedding. At 500 queries/day, even 20% repeats saves 100 Ollama calls
+const EMBED_CACHE_MAX = 200;
+const embedCache = new Map();
+function cacheKey(text) {
+    return text.trim().toLowerCase();
+}
+function evictOldest() {
+    if (embedCache.size <= EMBED_CACHE_MAX)
+        return;
+    let oldestKey = "";
+    let oldestTs = Infinity;
+    for (const [key, entry] of embedCache) {
+        if (entry.ts < oldestTs) {
+            oldestTs = entry.ts;
+            oldestKey = key;
+        }
+    }
+    if (oldestKey)
+        embedCache.delete(oldestKey);
+}
+// --- Embedding with Resilience ---
+// WHAT: Embed query text via Ollama with timeout, retry, and caching
+// WHY: Ollama is the bottleneck (~100-500ms). Cache eliminates redundant calls,
+//      timeout prevents hanging, retry handles transient failures
 async function embedQuery(text) {
-    const resp = await fetch(`${OLLAMA_URL}/api/embed`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ model: EMBED_MODEL, input: text }),
-    });
-    if (!resp.ok) {
-        throw new Error(`Ollama embed failed: ${resp.status} ${await resp.text()}`);
+    const key = cacheKey(text);
+    const cached = embedCache.get(key);
+    if (cached) {
+        cached.ts = Date.now(); // Touch for LRU
+        return cached.vec;
+    }
+    const vec = await embedWithRetry(text);
+    embedCache.set(key, { vec, ts: Date.now() });
+    evictOldest();
+    return vec;
+}
+async function embedWithRetry(text, retries = 1) {
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const resp = await fetch(`${OLLAMA_URL}/api/embed`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ model: EMBED_MODEL, input: text }),
+                signal: AbortSignal.timeout(10_000),
+            });
+            if (!resp.ok) {
+                throw new Error(`Ollama embed failed: ${resp.status} ${await resp.text()}`);
+            }
+            const data = (await resp.json());
+            return new Float32Array(data.embeddings[0]);
+        }
+        catch (err) {
+            const isLastAttempt = attempt === retries;
+            if (isLastAttempt) {
+                const message = err instanceof Error ? err.message : String(err);
+                throw new Error(`Embedding unavailable after ${retries + 1} attempt(s): ${message}. ` +
+                    "Ensure Ollama is running with the nomic-embed-text model loaded.");
+            }
+            // Wait before retry
+            await new Promise((resolve) => setTimeout(resolve, 500));
+        }
+    }
+    // Unreachable, but TypeScript needs it
+    throw new Error("Embedding failed");
+}
+// --- Prepared Statement Cache ---
+// WHAT: Cache the content_type lookup statement used in post-filtering
+// WHY: Avoid re-preparing the same statement on every search call
+let contentTypeStmt = null;
+function getContentTypeStmt(database) {
+    if (!contentTypeStmt) {
+        contentTypeStmt = database.prepare("SELECT content_type FROM articles WHERE url = ?");
     }
-    const data = (await resp.json());
-    return new Float32Array(data.embeddings[0]);
+    return contentTypeStmt;
 }
-export async function searchArticles(query, limit = 5) {
+// --- Helper: Post-filter by content type ---
+function filterByContentType(rows, contentType, database) {
+    if (!contentType)
+        return rows;
+    const stmt = getContentTypeStmt(database);
+    return rows.filter((row) => {
+        const article = stmt.get(row.article_url);
+        return article?.content_type === contentType;
+    });
+}
+// --- search_articles ---
+export async function searchArticles(query, limit = 5, contentType) {
     const database = getDb();
     const queryVec = await embedQuery(query);
+    // WHAT: Over-fetch when filtering by content_type, then post-filter
+    // WHY: sqlite-vec applies k before joins, so we can't pre-filter on content_type
+    const fetchK = contentType ? limit * 3 : limit;
     const rows = database
         .prepare(`
       SELECT
@@ -52,8 +138,9 @@ export async function searchArticles(query, limit = 5) {
         AND k = ?
       ORDER BY distance
     `)
-        .all(queryVec, limit);
-    return rows.map((row) => ({
+        .all(queryVec, fetchK);
+    const filtered = filterByContentType(rows, contentType, database);
+    return filtered.slice(0, limit).map((row) => ({
         title: row.article_title,
         section: row.section_heading,
         content: row.content,
@@ -62,9 +149,13 @@ export async function searchArticles(query, limit = 5) {
         relevance_score: Math.round((1 - row.distance / 2) * 1000) / 1000,
     }));
 }
-export async function getAnswer(question) {
+// --- get_answer ---
+export async function getAnswer(question, contentType) {
     const database = getDb();
     const queryVec = await embedQuery(question);
+    // WHAT: Over-fetch when filtering by content_type
+    // WHY: Same sqlite-vec k-before-join constraint as searchArticles
+    const fetchK = contentType ? 30 : 10;
     // Get the best matching chunk, preferring FAQ entries
     const rows = database
         .prepare(`
@@ -78,27 +169,29 @@ export async function getAnswer(question) {
       FROM vec_chunks
       LEFT JOIN chunks ON chunks.id = vec_chunks.chunk_id
       WHERE embedding MATCH ?
-        AND k = 10
+        AND k = ?
       ORDER BY distance
     `)
-        .all(queryVec);
-    if (rows.length === 0) {
+        .all(queryVec, fetchK);
+    const filteredRows = filterByContentType(rows, contentType, database);
+    if (filteredRows.length === 0) {
         return {
             answer: "No relevant information found in the ASTGL knowledge base.",
             source_title: "",
             source_url: "",
             related_articles: [],
+            confidence_score: 0,
         };
     }
     // Prefer FAQ entries if they're among the top results (within 20% of best distance)
-    const bestDistance = rows[0].distance;
-    const faqCandidate = rows.find((r) => r.chunk_type === "faq" && r.distance <= bestDistance * 1.2);
-    const best = faqCandidate || rows[0];
+    const bestDistance = filteredRows[0].distance;
+    const faqCandidate = filteredRows.find((r) => r.chunk_type === "faq" && r.distance <= bestDistance * 1.2);
+    const best = faqCandidate || filteredRows[0];
     // Collect related articles (unique, excluding the source)
     const seen = new Set();
     seen.add(best.article_url);
     const related = [];
-    for (const row of rows) {
+    for (const row of filteredRows) {
         if (!seen.has(row.article_url)) {
             seen.add(row.article_url);
             related.push({ title: row.article_title, url: row.article_url });
@@ -113,34 +206,176 @@ export async function getAnswer(question) {
         if (aMatch)
             answer = aMatch[1];
     }
+    // WHAT: Cosine distance 0-2 → confidence 0-1
+    // WHY: Gives consumers a normalized quality signal for the answer
+    const confidence = Math.round((1 - best.distance / 2) * 1000) / 1000;
     return {
         answer,
         source_title: best.article_title,
         source_url: best.article_url,
         related_articles: related,
+        confidence_score: confidence,
     };
 }
+// --- get_tutorial ---
+// WHAT: Find tutorial/guide content and return ordered steps from section headings
+// WHY: AI assistants asking "how do I X" benefit from structured step-by-step answers
+export async function getTutorial(query) {
+    const database = getDb();
+    const queryVec = await embedQuery(query);
+    const rows = database
+        .prepare(`
+      SELECT
+        chunks.article_title,
+        chunks.article_url,
+        chunks.section_heading,
+        chunks.content,
+        chunks.chunk_type,
+        chunks.article_order,
+        vec_chunks.distance
+      FROM vec_chunks
+      LEFT JOIN chunks ON chunks.id = vec_chunks.chunk_id
+      LEFT JOIN articles ON articles.url = chunks.article_url
+      WHERE embedding MATCH ?
+        AND k = 30
+      ORDER BY distance
+    `)
+        .all(queryVec);
+    if (rows.length === 0) {
+        return {
+            title: "",
+            url: "",
+            description: "No matching tutorials found.",
+            steps: [],
+            confidence_score: 0,
+        };
+    }
+    // WHAT: Pick the best-matching article and collect all its section chunks in order
+    // WHY: Tutorial steps come from the sequential sections of a single article
+    const bestUrl = rows[0].article_url;
+    const bestDistance = rows[0].distance;
+    const articleChunks = database
+        .prepare(`SELECT section_heading, content, chunk_type, article_order
+       FROM chunks
+       WHERE article_url = ? AND chunk_type = 'section'
+       ORDER BY article_order`)
+        .all(bestUrl);
+    const description = database
+        .prepare("SELECT description FROM articles WHERE url = ?")
+        .get(bestUrl);
+    const steps = articleChunks.map((c) => `**${c.section_heading}**\n${c.content}`);
+    return {
+        title: rows[0].article_title,
+        url: bestUrl,
+        description: description?.description || "",
+        steps,
+        confidence_score: Math.round((1 - bestDistance / 2) * 1000) / 1000,
+    };
+}
+// --- compare_topics ---
+// WHAT: Side-by-side comparison of two topics using vector search
+// WHY: "X vs Y" queries are common in AI-assisted research
+export async function compareTopics(topicA, topicB) {
+    const database = getDb();
+    const [vecA, vecB] = await Promise.all([
+        embedQuery(topicA),
+        embedQuery(topicB),
+    ]);
+    // WHAT: Get best-matching chunks for each topic independently
+    // WHY: Each side of the comparison needs its own top-ranked content
+    const getTopChunks = (vec) => database
+        .prepare(`
+        SELECT
+          chunks.article_title,
+          chunks.article_url,
+          chunks.section_heading,
+          chunks.content,
+          vec_chunks.distance
+        FROM vec_chunks
+        LEFT JOIN chunks ON chunks.id = vec_chunks.chunk_id
+        WHERE embedding MATCH ?
+          AND k = 5
+        ORDER BY distance
+      `)
+        .all(vec);
+    const rowsA = getTopChunks(vecA);
+    const rowsB = getTopChunks(vecB);
+    const buildSide = (rows) => {
+        if (rows.length === 0) {
+            return { title: "No matching content", url: "", key_points: [] };
+        }
+        // WHAT: Collect key points from top chunks, preferring different sections
+        // WHY: Diversity of sections gives a better comparison than repeated content
+        const seen = new Set();
+        const points = [];
+        for (const row of rows) {
+            const key = `${row.article_url}:${row.section_heading}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            const point = row.content.length > 300
+                ? row.content.slice(0, 300) + "..."
+                : row.content;
+            points.push(`**${row.section_heading}:** ${point}`);
+            if (points.length >= 3)
+                break;
+        }
+        return {
+            title: rows[0].article_title,
+            url: rows[0].article_url,
+            key_points: points,
+        };
+    };
+    const worstDistance = Math.max(rowsA[0]?.distance ?? 2, rowsB[0]?.distance ?? 2);
+    return {
+        topic_a: buildSide(rowsA),
+        topic_b: buildSide(rowsB),
+        confidence_score: Math.round((1 - worstDistance / 2) * 1000) / 1000,
+    };
+}
+// --- get_latest ---
+// WHAT: Return most recently added articles
+// WHY: "What's new" queries help users discover fresh content
+export function getLatest(limit = 5) {
+    const database = getDb();
+    // WHAT: Order by processed_at (structuring pipeline) then rowid (ingest order)
+    // WHY: Original ingested articles lack processed_at; rowid is the fallback
+    const rows = database
+        .prepare(`SELECT title, description, url, content_type, processed_at
+       FROM articles
+       ORDER BY COALESCE(processed_at, '') DESC, rowid DESC
+       LIMIT ?`)
+        .all(limit);
+    return rows.map((r) => ({
+        title: r.title,
+        description: r.description,
+        url: r.url,
+        content_type: r.content_type || "article",
+        added_at: r.processed_at,
+    }));
+}
+// --- list_topics ---
+// WHAT: List all topics with their content types and section headings
+// WHY: Gives AI assistants an overview of available coverage
 export function listTopics() {
     const database = getDb();
-    const articles = database
-        .prepare("SELECT title, description, url FROM articles ORDER BY rowid")
-        .all();
-    // Extract topic keywords from section headings for each article
-    const sectionsByUrl = new Map();
-    const sections = database
-        .prepare("SELECT DISTINCT article_url, section_heading FROM chunks WHERE chunk_type = 'section'")
+    // WHAT: Single query with GROUP_CONCAT instead of two queries + in-memory join
+    // WHY: Reduces to one DB round-trip and eliminates the Map construction overhead
+    const rows = database
+        .prepare(`SELECT a.title, a.description, a.url,
+              COALESCE(a.content_type, 'article') as content_type,
+              GROUP_CONCAT(DISTINCT c.section_heading) as sections
+       FROM articles a
+       LEFT JOIN chunks c ON c.article_url = a.url AND c.chunk_type = 'section'
+       GROUP BY a.url
+       ORDER BY a.rowid`)
         .all();
-    for (const s of sections) {
-        if (!sectionsByUrl.has(s.article_url)) {
-            sectionsByUrl.set(s.article_url, []);
-        }
-        sectionsByUrl.get(s.article_url).push(s.section_heading);
-    }
-    return articles.map((a) => ({
-        title: a.title,
-        description: a.description,
-        url: a.url,
-        topics: sectionsByUrl.get(a.url) || [],
+    return rows.map((r) => ({
+        title: r.title,
+        description: r.description,
+        url: r.url,
+        content_type: r.content_type,
+        topics: r.sections ? r.sections.split(",") : [],
     }));
 }
 //# sourceMappingURL=search.js.map