@sarjallab09/figma-intelligence 1.0.1 → 1.2.0

package/figma-bridge-plugin/content-context.js

@@ -1,869 +0,0 @@
-#!/usr/bin/env node
-/**
- * content-context.js — Universal content parsing for knowledge grounding.
- *
- * Extracts text from PDFs, DOCX files, web URLs, and plain text.
- * The extracted content is stored as "knowledge sources" and injected
- * as grounding context into any AI provider's chat messages.
- *
- * Supported formats:
- *   - PDF  → pdf-parse (pure JS, no native binaries)
- *   - DOCX → mammoth (pure JS, JSZip-based)
- *   - URL  → cheerio + @mozilla/readability
- *   - TXT/MD/CSV/JSON → plain UTF-8 text
- */
-
-const https = require("https");
-const http = require("http");
-const fs = require("fs");
-const path = require("path");
-
-// Knowledge Hub directory (sibling to this file)
-const KNOWLEDGE_HUB_DIR = path.join(__dirname, "knowledge-hub");
-
-// Lazy-load heavy dependencies to keep startup fast
-let _pdfParse, _mammoth, _cheerio, _Readability, _JSDOM;
-
-function getPdfParse() {
-  if (!_pdfParse) _pdfParse = require("pdf-parse"); // exports { PDFParse, ... }
-  return _pdfParse;
-}
-function getMammoth() {
-  if (!_mammoth) _mammoth = require("mammoth");
-  return _mammoth;
-}
-function getCheerio() {
-  if (!_cheerio) _cheerio = require("cheerio");
-  return _cheerio;
-}
-function getReadability() {
-  if (!_Readability) _Readability = require("@mozilla/readability");
-  return _Readability;
-}
-
-// ── PDF Parsing ─────────────────────────────────────────────────────────────
-
-/**
- * Extract text from a PDF buffer.
- * @param {Buffer} buffer — raw PDF bytes
- * @returns {Promise<{ title: string, text: string, pages: number }>}
- */
-async function parsePdfBuffer(buffer) {
-  const { PDFParse } = getPdfParse();
-  // pdf-parse v2+ requires Uint8Array, not Buffer
-  const uint8 = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
-  const parser = new PDFParse(uint8);
-  const result = await parser.getText();
-  // result = { pages: [{ text, num }], text: string, total: number }
-  // Clean up: join page texts, remove page markers like "-- 1 of 320 --"
-  const rawText = result.text || result.pages.map(p => p.text).join("\n\n");
-  const fullText = rawText.replace(/\n-- \d+ of \d+ --\n/g, "\n\n").trim();
-  let title = "";
-  // Try to get title from info
-  try {
-    const info = await parser.getInfo();
-    title = info?.Title || info?.title || "";
-  } catch {}
-  return {
-    title,
-    text: fullText,
-    pages: result.total || result.pages?.length || 0,
-  };
-}
-
-// ── DOCX Parsing ────────────────────────────────────────────────────────────
-
-/**
- * Extract text from a DOCX buffer.
- * @param {Buffer} buffer — raw DOCX bytes
- * @returns {Promise<{ title: string, text: string }>}
- */
-async function parseDocxBuffer(buffer) {
-  const mammoth = getMammoth();
-  const result = await mammoth.extractRawText({ buffer });
-  // Try to derive title from first line
-  const firstLine = (result.value || "").split("\n").find(l => l.trim().length > 0) || "";
-  return {
-    title: firstLine.length > 5 && firstLine.length < 120 ? firstLine.trim() : "",
-    text: result.value || "",
-  };
-}
-
-// ── URL Content Extraction ──────────────────────────────────────────────────
-
-/**
- * Fetch a URL and extract its readable text content.
- * @param {string} url — any HTTP/HTTPS URL
- * @returns {Promise<{ title: string, text: string, url: string }>}
- */
-async function fetchUrlContent(url) {
-  const html = await fetchHtml(url);
-  const cheerio = getCheerio();
-  const { Readability } = getReadability();
-
-  const $ = cheerio.load(html);
-
-  // Remove scripts, styles, nav, footer, ads
-  $("script, style, nav, footer, aside, [role='banner'], [role='navigation'], .ad, .ads, .advertisement").remove();
-
-  // Try @mozilla/readability first (needs a DOM-like object)
-  // Since Readability expects a DOM, we create a minimal one from cheerio
-  let title = $("title").text().trim() || $("h1").first().text().trim() || "";
-  let text = "";
-
-  // Extract text from main content areas
-  const mainSelectors = ["main", "article", "[role='main']", ".content", ".post-content", ".entry-content", "#content"];
-  for (const sel of mainSelectors) {
-    const el = $(sel);
-    if (el.length && el.text().trim().length > 100) {
-      text = el.text().trim();
-      break;
-    }
-  }
-
-  // Fallback: get body text
-  if (!text) {
-    text = $("body").text().trim();
-  }
-
-  // Clean up whitespace
-  text = text.replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim();
-
-  // Truncate very long pages
-  if (text.length > 50000) {
-    text = text.slice(0, 50000) + "\n\n[Content truncated at 50,000 characters]";
-  }
-
-  return { title, text, url };
-}
-
-function fetchHtml(url, maxRedirects = 5) {
-  return new Promise((resolve, reject) => {
-    if (maxRedirects <= 0) return reject(new Error("Too many redirects"));
-    const mod = url.startsWith("https") ? https : http;
-    const req = mod.get(url, {
-      headers: {
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.9",
-      },
-    }, (res) => {
-      if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) {
-        const next = res.headers.location.startsWith("http")
-          ? res.headers.location
-          : new URL(res.headers.location, url).href;
-        return resolve(fetchHtml(next, maxRedirects - 1));
-      }
-      if (res.statusCode !== 200 && res.statusCode !== 202) return reject(new Error(`HTTP ${res.statusCode}`));
-      let body = "";
-      res.on("data", (chunk) => { body += chunk.toString(); });
-      res.on("end", () => resolve(body));
-      res.on("error", reject);
-    });
-    req.on("error", reject);
-    req.setTimeout(15000, () => { req.destroy(); reject(new Error("Request timed out")); });
-  });
-}
-
-// ── Content Source Management ───────────────────────────────────────────────
-
-/**
- * Create a content source entry from extracted text.
- */
-function createContentSource(title, text, meta) {
-  const id = "ks-" + Date.now() + "-" + Math.random().toString(36).slice(2, 6);
-  return {
-    id,
-    title: title || "Untitled",
-    sources: [{
-      name: title || "Content",
-      content: text,
-    }],
-    meta: meta || {},  // { pages, url, fileType, fileName }
-    extractedAt: new Date().toISOString(),
-  };
-}
-
-/**
- * Extract keywords from a query string, including bigrams for compound concepts.
- * E.g., "design system components" → ["design system", "design", "system", "components"]
- */
-function extractKeywords(query) {
-  const words = (query || "")
-    .toLowerCase()
-    .replace(/[^\w\s-]/g, " ")
-    .split(/\s+/)
-    .filter((w) => w.length > 2 && !STOP_WORDS.has(w));
-
-  // Generate bigrams for multi-word concepts
-  const bigrams = [];
-  for (let i = 0; i < words.length - 1; i++) {
-    bigrams.push(words[i] + " " + words[i + 1]);
-  }
-
-  // Return bigrams first (higher specificity), then unigrams
-  return [...bigrams, ...words];
-}
-
-/**
- * Score a text block against keywords using TF-IDF-like weighting.
- * Bigrams score 4x (more specific), word-boundary matches score 2x,
- * substring matches score 1x. Multiple occurrences add diminishing returns.
- */
-function scoreText(text, keywords) {
-  const lower = text.toLowerCase();
-  let score = 0;
-  let matchedCount = 0;
-  const uniqueMatches = new Set();
-
-  for (const kw of keywords) {
-    if (!lower.includes(kw)) continue;
-
-    const isBigram = kw.includes(" ");
-    const regex = new RegExp(`\\b${kw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "gi");
-    const matches = text.match(regex);
-
-    if (matches) {
-      // Count unique keyword roots matched
-      if (isBigram) {
-        kw.split(" ").forEach((w) => uniqueMatches.add(w));
-      } else {
-        uniqueMatches.add(kw);
-      }
-
-      // Score: bigrams are worth more (more specific match)
-      const baseScore = isBigram ? 4 : 2;
-      // Diminishing returns for repeated matches: 1st=full, 2nd=half, 3rd+=quarter
-      const occurrences = Math.min(matches.length, 5);
-      score += baseScore + Math.min(occurrences - 1, 3) * (baseScore * 0.25);
-    } else {
-      // Substring match (less precise)
-      uniqueMatches.add(isBigram ? kw.split(" ")[0] : kw);
-      score += isBigram ? 2 : 0.5;
-    }
-  }
-
-  matchedCount = uniqueMatches.size;
-  return { score, matchedCount };
-}
-
-/**
- * Extract the most relevant passages from content using keyword scoring.
- * Returns a string of the top passages within the char budget.
- */
-function extractRelevantPassages(content, keywords, charBudget) {
-  if (!keywords || keywords.length === 0) {
-    // No keywords — fall back to truncation
-    return content.length > charBudget
-      ? content.slice(0, charBudget) + "\n…[truncated]"
-      : content;
-  }
-
-  const paragraphs = content.split(/\n{2,}/).filter((p) => p.trim().length > 30);
-  if (paragraphs.length === 0) {
-    return content.slice(0, charBudget);
-  }
-
-  // Score each paragraph
-  const scored = paragraphs.map((para) => {
-    const { score, matchedCount } = scoreText(para, keywords);
-    return { para: para.trim(), score, matchedCount };
-  });
-
-  // Sort by score descending, keep top paragraphs within budget
-  scored.sort((a, b) => b.score - a.score);
-
-  const selected = [];
-  let usedChars = 0;
-  for (const { para, score } of scored) {
-    if (score === 0 && selected.length > 0) break; // Stop adding irrelevant paragraphs
-    if (usedChars + para.length > charBudget) {
-      if (selected.length === 0) {
-        // At least include a truncated version of the best paragraph
-        selected.push(para.slice(0, charBudget) + "…");
-      }
-      break;
-    }
-    selected.push(para);
-    usedChars += para.length;
-  }
-
-  return selected.join("\n\n");
-}
-
-/**
- * Build a grounding context string from active content sources.
- * Prepended to chat messages before routing to the AI provider.
- *
- * Optimizations:
- * - Relevance filtering: only includes sources with keyword overlap
- * - Adaptive truncation: extracts best paragraphs, not just first N chars
- * - Budget-based: total grounding limited to ~4000 tokens (~16000 chars)
- */
-const GROUNDING_CHAR_BUDGET = 16000; // ~4000 tokens total
-
-function buildGroundingContext(contentSources, userQuery) {
-  if (!contentSources || contentSources.size === 0) return "";
-
-  const keywords = extractKeywords(userQuery);
-
-  // For chunked sources: score individual chunks and pick the best ones globally
-  // For non-chunked sources: sample multiple positions (not just first 2000 chars)
-  const allScoredChunks = []; // { sourceTitle, chunkTitle, content, score, meta }
-
-  for (const [id, data] of contentSources) {
-    if (data.isChunked && data.sources.length > 1) {
-      // Chunked source: score each chunk individually
-      // Extract unigrams for pre-filter (bigrams are checked in full scoring)
-      const unigrams = keywords.filter((k) => !k.includes(" "));
-
-      for (const src of data.sources) {
-        if (!src.content) continue;
-        // Pre-filter: check if any unigrams match stored chunk keywords
-        let preScore = 0;
-        if (src.keywords && unigrams.length > 0) {
-          for (const kw of unigrams) {
-            // Exact match on stored keywords (not substring)
-            if (src.keywords.includes(kw)) {
-              preScore += 2;
-            } else if (src.keywords.some((ck) => ck.includes(kw))) {
-              preScore += 0.5;
-            }
-          }
-        }
-        // Full text scoring for chunks that passed pre-filter
-        if (preScore > 0 || keywords.length === 0) {
-          const { score: textScore, matchedCount } = scoreText(src.content.slice(0, 4000), keywords);
-          const totalScore = preScore + textScore;
-          if (totalScore > 0 || keywords.length === 0) {
-            allScoredChunks.push({
-              sourceTitle: data.title,
-              chunkTitle: src.title || data.title,
-              content: src.content,
-              score: totalScore,
-              matchedCount,
-              meta: data.meta,
-            });
-          }
-        }
-      }
-    } else {
-      // Non-chunked source: sample multiple positions across the content
-      for (const src of data.sources) {
-        if (!src.content) continue;
-        let totalScore = 0;
-        const len = src.content.length;
-        const sampleSize = 2000;
-        const positions = [0, Math.floor(len * 0.25), Math.floor(len * 0.5), Math.floor(len * 0.75)];
-        for (const pos of positions) {
-          const { score } = scoreText(src.content.slice(pos, pos + sampleSize), keywords);
-          totalScore += score;
-        }
-        allScoredChunks.push({
-          sourceTitle: data.title,
-          chunkTitle: data.title,
-          content: src.content,
-          score: totalScore,
-          meta: data.meta,
-        });
-      }
-    }
-  }
-
-  if (allScoredChunks.length === 0) return "";
-
-  // Sort all chunks by score, take the best ones within budget
-  allScoredChunks.sort((a, b) => b.score - a.score);
-
-  // Filter to relevant chunks (score > 0) if we have keywords
-  const relevant = keywords.length > 0
-    ? allScoredChunks.filter((c) => c.score > 0)
-    : allScoredChunks;
-
-  const chunksToInclude = relevant.length > 0 ? relevant : allScoredChunks.slice(0, 5);
-
-  const parts = [];
-  parts.push("=== KNOWLEDGE CONTEXT ===");
-  parts.push("The following excerpts are from the user's knowledge library. Use them to provide accurate, well-sourced answers. Synthesize the information — do not dump raw text.\n");
-
-  let usedChars = 0;
-  let currentSource = "";
-
-  for (const chunk of chunksToInclude) {
-    if (usedChars >= GROUNDING_CHAR_BUDGET) break;
-
-    // Add source header if it changed
-    if (chunk.sourceTitle !== currentSource) {
-      const metaInfo = [];
-      if (chunk.meta?.url) metaInfo.push(chunk.meta.url);
-      const metaStr = metaInfo.length > 0 ? ` (${metaInfo.join(", ")})` : "";
-      parts.push(`\n--- "${chunk.sourceTitle}"${metaStr} ---`);
-      currentSource = chunk.sourceTitle;
-    }
-
-    // Add chunk heading if different from source title
-    if (chunk.chunkTitle && chunk.chunkTitle !== chunk.sourceTitle) {
-      parts.push(`\n### ${chunk.chunkTitle}`);
-    }
-
-    const remaining = GROUNDING_CHAR_BUDGET - usedChars;
-    const passage = extractRelevantPassages(chunk.content, keywords, remaining);
-    parts.push(passage);
-    usedChars += passage.length;
-  }
-
-  parts.push("\n=== END KNOWLEDGE CONTEXT ===\n");
-  return parts.join("\n");
-}
-
-// ── Knowledge Hub ───────────────────────────────────────────────────────────
-
-// Cache of parsed hub files: fileName → { id, title, text, meta, parsedAt }
-const _hubCache = new Map();
-
-/**
- * Scan the knowledge-hub/ folder and return a catalog of available files.
- * Does NOT parse content — just lists what's available.
- */
-function scanKnowledgeHub() {
-  if (!fs.existsSync(KNOWLEDGE_HUB_DIR)) return [];
-
-  const files = fs.readdirSync(KNOWLEDGE_HUB_DIR);
-  const supported = [".pdf", ".docx", ".doc", ".txt", ".md", ".csv", ".json", ".xml", ".html"];
-
-  return files
-    .filter((f) => {
-      if (f.startsWith(".")) return false;
-      // Prefer .chunks.json — hide the original PDF if a chunks file exists
-      if (f.endsWith(".chunks.json")) return true;
-      const ext = path.extname(f).toLowerCase();
-      if (!supported.includes(ext)) return false;
-      // Hide PDFs that have been converted to chunks
-      if (ext === ".pdf") {
-        const slug = f.replace(/\.pdf$/i, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
-        if (files.includes(`${slug}.chunks.json`)) return false;
-      }
-      return true;
-    })
-    .map((f) => {
-      const isChunks = f.endsWith(".chunks.json");
-      const ext = isChunks ? "chunks" : path.extname(f).toLowerCase().replace(".", "");
-      const stat = fs.statSync(path.join(KNOWLEDGE_HUB_DIR, f));
-      const cached = _hubCache.get(f);
-
-      let title = f.replace(/\.chunks\.json$/, "").replace(/\.\w+$/, "").replace(/[-_]/g, " ");
-      let charCount = cached ? cached.totalChars || cached.text?.length : null;
-      let preview = cached ? (cached.preview || (cached.text || "").slice(0, 200)).replace(/\s+/g, " ").trim() : null;
-
-      // For chunks files, read title from the JSON metadata
-      if (isChunks && !cached) {
-        try {
-          const meta = JSON.parse(fs.readFileSync(path.join(KNOWLEDGE_HUB_DIR, f), "utf8"));
-          title = meta.title || title;
-          charCount = meta.totalChars || null;
-          preview = meta.chunks?.[0]?.text?.slice(0, 200)?.replace(/\s+/g, " ")?.trim() || null;
-        } catch {}
-      } else if (isChunks && cached) {
-        title = cached.title || title;
-      }
-
-      return {
-        fileName: f,
-        fileType: ext,
-        title,
-        sizeBytes: stat.size,
-        cached: !!cached,
-        preview,
-        charCount,
-      };
-    });
-}
-
-/**
- * Load and parse a specific file from the knowledge hub.
- * Uses cache to avoid re-parsing on every request.
- * Returns a content source ready to add to activeContentSources.
- */
-async function loadHubFile(fileName) {
-  const filePath = path.join(KNOWLEDGE_HUB_DIR, fileName);
-  if (!fs.existsSync(filePath)) {
-    throw new Error(`File not found in knowledge hub: ${fileName}`);
-  }
-
-  // Check cache (invalidate if file changed)
-  const stat = fs.statSync(filePath);
-  const cached = _hubCache.get(fileName);
-  if (cached && cached.mtime === stat.mtimeMs) {
-    if (cached.chunks) {
-      // Return chunked source (multi-source for chunk-level scoring)
-      return createChunkedContentSource(cached.title, cached.chunks, cached.meta);
-    }
-    return createContentSource(cached.title, cached.text, cached.meta);
-  }
-
-  // Handle .chunks.json files (pre-indexed, instant load)
-  if (fileName.endsWith(".chunks.json")) {
-    const data = JSON.parse(fs.readFileSync(filePath, "utf8"));
-    const meta = {
-      fileName,
-      fileType: "chunks",
-      hubFile: true,
-      pages: data.pages,
-      source: data.source,
-      chunkCount: data.chunks.length,
-    };
-
-    // Cache the chunks
-    _hubCache.set(fileName, {
-      title: data.title,
-      chunks: data.chunks,
-      totalChars: data.totalChars,
-      preview: data.chunks[0]?.text?.slice(0, 200) || "",
-      meta,
-      mtime: stat.mtimeMs,
-    });
-
-    return createChunkedContentSource(data.title, data.chunks, meta);
-  }
-
-  // Parse the file (PDF, DOCX, plain text)
-  const buffer = fs.readFileSync(filePath);
-  const ext = path.extname(fileName).toLowerCase();
-  let title, text;
-  const meta = { fileName, fileType: ext.replace(".", ""), hubFile: true };
-
-  if (ext === ".pdf") {
-    const result = await parsePdfBuffer(buffer);
-    title = result.title || fileName.replace(/\.\w+$/, "");
-    text = result.text;
-    meta.pages = result.pages;
-  } else if (ext === ".docx" || ext === ".doc") {
-    const result = await parseDocxBuffer(buffer);
-    title = result.title || fileName.replace(/\.\w+$/, "");
-    text = result.text;
-  } else {
-    title = fileName.replace(/\.\w+$/, "").replace(/[-_]/g, " ");
-    text = buffer.toString("utf-8");
-  }
-
-  if (!text || text.trim().length === 0) {
-    throw new Error(`No text content could be extracted from: ${fileName}`);
-  }
-
-  // Cache the parsed result
-  _hubCache.set(fileName, { title, text, meta, mtime: stat.mtimeMs });
-
-  return createContentSource(title, text, meta);
-}
-
-/**
- * Create a content source from pre-chunked data.
- * Each chunk becomes a separate source entry for chunk-level scoring.
- */
-function createChunkedContentSource(title, chunks, meta) {
-  return {
-    id: "hub-" + Date.now() + "-" + Math.random().toString(36).slice(2, 6),
-    title,
-    sources: chunks.map((chunk) => ({
-      title: chunk.heading || title,
-      content: chunk.text,
-      keywords: chunk.keywords || [],
-    })),
-    meta: meta || {},
-    extractedAt: new Date().toISOString(),
-    isChunked: true,
-  };
-}
-
-/**
- * Search the knowledge hub for files relevant to a query.
- * Simple keyword matching against file names and cached content.
- */
-function searchHub(query) {
-  const catalog = scanKnowledgeHub();
-  if (!query || !query.trim()) return catalog;
-
-  const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
-  return catalog
-    .map((item) => {
-      const haystack = (item.title + " " + item.fileName + " " + (item.preview || "")).toLowerCase();
-      const score = terms.reduce((s, t) => s + (haystack.includes(t) ? 1 : 0), 0);
-      return { ...item, score };
-    })
-    .filter((item) => item.score > 0)
-    .sort((a, b) => b.score - a.score);
-}
-
-// ── Tier 1: Knowledge Hub Instant Answer ─────────────────────────────────────
-
-const STOP_WORDS = new Set([
-  "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
-  "have", "has", "had", "do", "does", "did", "will", "would", "could",
-  "should", "may", "might", "shall", "can", "need", "must",
-  "i", "me", "my", "we", "our", "you", "your", "he", "she", "it",
-  "they", "them", "their", "this", "that", "these", "those",
-  "and", "or", "but", "not", "no", "nor", "so", "if", "then",
-  "of", "in", "on", "at", "to", "for", "with", "by", "from",
-  "about", "into", "through", "during", "before", "after",
-  "above", "below", "between", "under", "over",
-  "just", "also", "very", "too", "more", "most", "some", "any",
-  "all", "each", "every", "both", "few", "many", "much",
-  "tell", "explain", "describe", "show",
-]);
-
-/**
- * Search loaded content sources for a direct answer to a query.
- * Scores ALL paragraphs globally, returns the top matches with surrounding context.
- */
-function searchContentForAnswer(query, contentSources) {
-  if (!query || !contentSources || contentSources.size === 0) return null;
-
-  const keywords = extractKeywords(query);
-  if (keywords.length === 0) return null;
-
-  // Count only unique unigrams for threshold (bigrams are bonus)
-  const uniqueKeywords = keywords.filter((k) => !k.includes(" "));
-  const minMatches = Math.max(2, Math.ceil(uniqueKeywords.length * 0.4));
-
-  // Score ALL paragraphs globally
-  const candidates = [];
-
-  for (const [, source] of contentSources) {
-    for (const src of source.sources) {
-      if (!src.content) continue;
-      const paragraphs = src.content.split(/\n{2,}/).filter((p) => p.trim().length > 40);
-
-      for (let pi = 0; pi < paragraphs.length; pi++) {
-        const para = paragraphs[pi];
-        const { score, matchedCount } = scoreText(para, keywords);
-
-        if (matchedCount >= minMatches) {
-          // Include surrounding context (previous + next paragraph) for better answers
-          const contextParts = [];
-          if (pi > 0 && paragraphs[pi - 1].trim().length > 30) {
-            contextParts.push(paragraphs[pi - 1].trim());
-          }
-          contextParts.push(para.trim());
-          if (pi < paragraphs.length - 1 && paragraphs[pi + 1].trim().length > 30) {
-            contextParts.push(paragraphs[pi + 1].trim());
-          }
-
-          candidates.push({
-            text: contextParts.join("\n\n"),
-            mainPara: para.trim(),
-            score,
-            matchedCount,
-            title: src.title || source.title,
-            sourceTitle: source.title,
-            meta: source.meta,
-          });
-        }
-      }
-    }
-  }
-
-  if (candidates.length === 0) return null;
-
-  // Sort by score descending, take top 3 for a richer answer
-  candidates.sort((a, b) => b.score - a.score);
-  const topMatches = candidates.slice(0, 3);
-
-  // Deduplicate overlapping text
-  const seen = new Set();
-  const uniqueMatches = topMatches.filter((m) => {
-    const key = m.mainPara.slice(0, 100);
-    if (seen.has(key)) return false;
-    seen.add(key);
-    return true;
-  });
-
-  // Format the answer
-  const parts = [];
-  for (const match of uniqueMatches) {
-    const snippet = match.text.length > 1200
-      ? match.text.slice(0, 1200) + "..."
-      : match.text;
-
-    const heading = match.title !== match.sourceTitle
-      ? `**From "${match.sourceTitle}" — ${match.title}**`
-      : `**From "${match.sourceTitle}"**`;
-
-    parts.push(`${heading}:\n\n> ${snippet}`);
-  }
-
-  const metaInfo = [];
-  if (uniqueMatches[0].meta?.fileType) metaInfo.push(uniqueMatches[0].meta.fileType.toUpperCase());
-  if (uniqueMatches[0].meta?.pages) metaInfo.push(`${uniqueMatches[0].meta.pages} pages`);
-  const metaStr = metaInfo.length > 0 ? ` (${metaInfo.join(", ")})` : "";
-
-  return {
-    text: parts.join("\n\n---\n\n") + `\n\n_Sources: ${[...new Set(uniqueMatches.map((m) => `"${m.sourceTitle}"`))].join(", ")}${metaStr}_`,
-    sources: uniqueMatches.map((m) => ({ title: m.sourceTitle, meta: m.meta })),
-  };
-}
-
-// ── Tier 2: Web Reference Search ─────────────────────────────────────────────
-
-// Default reference sites — starts empty, users add via UI
-// NN/g is suggested in the UI but not auto-enabled to avoid blocking chat
-let referenceSites = [];
-
-// Article cache: url → { title, text, fetchedAt }
-const _articleCache = new Map();
-const ARTICLE_CACHE_TTL = 60 * 60 * 1000; // 1 hour
-
-function getReferenceSites() {
-  return [...referenceSites];
-}
-
-function addReferenceSite(site) {
-  const id = "ref-" + Date.now() + "-" + Math.random().toString(36).slice(2, 6);
-  const entry = {
-    id,
-    name: site.name || site.searchDomain || "Reference",
-    baseUrl: site.baseUrl || site.url || "",
-    searchDomain: site.searchDomain || extractDomain(site.baseUrl || site.url || ""),
-  };
-  referenceSites.push(entry);
-  return entry;
-}
-
-function removeReferenceSite(id) {
-  referenceSites = referenceSites.filter((s) => s.id !== id);
-}
-
-function extractDomain(url) {
-  try {
-    return new URL(url).hostname.replace(/^www\./, "");
-  } catch {
-    return url.replace(/^https?:\/\/(www\.)?/, "").split("/")[0];
-  }
-}
-
-/**
- * Search configured reference sites for articles relevant to a query.
- * Uses DuckDuckGo HTML search (no API key needed).
- * Returns a formatted answer with link, or null if no match.
- */
-async function searchReferenceSites(query) {
-  if (!query || referenceSites.length === 0) return null;
-
-  // Try each reference site
-  for (const site of referenceSites) {
-    try {
-      const result = await searchSingleSite(query, site);
-      if (result) return result;
-    } catch (err) {
-      console.error(`[web-ref] Error searching ${site.name}: ${err.message}`);
-    }
-  }
-  return null;
-}
-
-async function searchSingleSite(query, site) {
-  const searchQuery = `site:${site.searchDomain} ${query}`;
-  const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}`;
-
-  // Fetch DuckDuckGo search results
-  let html;
-  try {
-    html = await fetchHtml(searchUrl);
-  } catch (err) {
-    console.error(`[web-ref] DuckDuckGo search failed: ${err.message}`);
-    return null;
-  }
-
-  // Parse results to extract article URLs
-  const cheerio = getCheerio();
-  const $ = cheerio.load(html);
-  const resultLinks = [];
-
-  $("a.result__a").each((i, el) => {
-    const href = $(el).attr("href") || "";
-    const title = $(el).text().trim();
-    // DuckDuckGo wraps URLs in a redirect — extract the actual URL
-    const urlMatch = href.match(/uddg=([^&]+)/);
-    const actualUrl = urlMatch ? decodeURIComponent(urlMatch[1]) : href;
-    if (actualUrl.includes(site.searchDomain) && title) {
-      resultLinks.push({ url: actualUrl, title });
-    }
-  });
-
-  if (resultLinks.length === 0) return null;
-
-  const topResult = resultLinks[0];
-
-  // Check article cache
-  const cached = _articleCache.get(topResult.url);
-  if (cached && Date.now() - cached.fetchedAt < ARTICLE_CACHE_TTL) {
-    return formatWebAnswer(cached.title, cached.text, topResult.url, site.name);
-  }
-
-  // Fetch and extract article content
-  try {
-    const article = await fetchUrlContent(topResult.url);
-    const articleText = article.text.slice(0, 3000); // Truncate for answer
-    const articleTitle = article.title || topResult.title;
-
-    // Cache the article
-    _articleCache.set(topResult.url, {
-      title: articleTitle,
-      text: articleText,
-      fetchedAt: Date.now(),
-    });
-
-    return formatWebAnswer(articleTitle, articleText, topResult.url, site.name);
-  } catch (err) {
-    console.error(`[web-ref] Failed to fetch article: ${err.message}`);
-    return null;
-  }
-}
-
-function formatWebAnswer(title, text, url, siteName) {
-  // Extract the most relevant paragraphs (first ~1500 chars)
-  const excerpt = text.length > 1500 ? text.slice(0, 1500) + "..." : text;
-
-  return {
-    text: `**From ${siteName}:**\n\n**[${title}](${url})**\n\n> ${excerpt}\n\n_[Read full article](${url})_`,
-    url,
-    title,
-    siteName,
-  };
-}
-
-/**
- * Pre-warm the hub cache by loading all .chunks.json files.
- * Call on relay startup for instant first-query response.
- */
-async function prewarmHub() {
-  const catalog = scanKnowledgeHub();
-  let loaded = 0;
-  for (const file of catalog) {
-    if (file.fileName.endsWith(".chunks.json")) {
-      try {
-        await loadHubFile(file.fileName);
-        loaded++;
-      } catch {}
-    }
-  }
-  return loaded;
-}
-
-module.exports = {
-  parsePdfBuffer,
-  parseDocxBuffer,
-  fetchUrlContent,
-  createContentSource,
-  createChunkedContentSource,
-  buildGroundingContext,
-  scanKnowledgeHub,
-  loadHubFile,
-  searchHub,
-  searchContentForAnswer,
-  searchReferenceSites,
-  getReferenceSites,
-  addReferenceSite,
-  removeReferenceSite,
-  prewarmHub,
-  KNOWLEDGE_HUB_DIR,
-};