llm-wiki-kit 0.1.5 → 0.1.6

package/src/wiki-search.js

@@ -0,0 +1,243 @@
+import { join } from 'path';
+import { readText } from './fs-utils.js';
+import { redactText } from './redaction.js';
+import {
+  buildWikiGraph,
+  collectWikiPages,
+  pageLookup,
+  readMemoryExcerpt,
+} from './wiki-model.js';
+
+const DEFAULT_LIMIT = 5;
+const SNIPPET_CHARS = 350;
+let miniSearchLoader = null;
+
+async function loadMiniSearch() {
+  if (!miniSearchLoader) {
+    miniSearchLoader = import('minisearch')
+      .then((module) => module.default || module.MiniSearch || null)
+      .catch(() => null);
+  }
+  return miniSearchLoader;
+}
+
+function normalizeText(value) {
+  return String(value || '').normalize('NFC').toLowerCase();
+}
+
+export function tokenizeWikiQuery(query) {
+  return normalizeText(query)
+    .replace(/[^\p{Letter}\p{Number}]+/gu, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length >= 2)
+    .slice(0, 80);
+}
+
+function substringScore(page, terms) {
+  if (terms.length === 0) return 0;
+  const haystack = normalizeText(`${page.title}\n${page.rel}\n${page.body}`);
+  let score = 0;
+  for (const term of terms) {
+    if (haystack.includes(term)) score += term.length > 3 ? 2 : 1;
+  }
+  return score;
+}
+
+function snippetFor(page, terms) {
+  const text = page.body.replace(/\s+/g, ' ').trim() || page.content.replace(/\s+/g, ' ').trim();
+  const lower = normalizeText(text);
+  let start = 0;
+  for (const term of terms) {
+    const index = lower.indexOf(term);
+    if (index !== -1) {
+      start = Math.max(0, index - 80);
+      break;
+    }
+  }
+  return text.slice(start, start + SNIPPET_CHARS);
+}
+
+function resultRecord(page, score, fields = {}) {
+  return {
+    path: page.rel,
+    title: page.title,
+    type: page.type || '',
+    memoryType: page.memoryType || '',
+    score,
+    directScore: fields.directScore || 0,
+    linkScore: fields.linkScore || 0,
+    source: fields.source || 'direct',
+    via: fields.via || [],
+    matchedTerms: fields.matchedTerms || [],
+    snippet: fields.snippet || '',
+  };
+}
+
+function sortHits(a, b) {
+  if (b.score !== a.score) return b.score - a.score;
+  if (a.source !== b.source) return a.source === 'direct' ? -1 : 1;
+  return a.path.localeCompare(b.path);
+}
+
+export async function searchWiki(projectRoot, query, options = {}) {
+  return (await performSearch(projectRoot, query, options)).hits;
+}
+
+async function performSearch(projectRoot, query, options = {}) {
+  const opts = typeof options === 'number' ? { limit: options } : options;
+  const limit = Number(opts.limit || DEFAULT_LIMIT);
+  const expand = opts.expand !== false;
+  const terms = tokenizeWikiQuery(query);
+  if (!query || terms.length === 0) {
+    return { hits: [], search: 'none' };
+  }
+
+  let pages = [];
+  try {
+    pages = await collectWikiPages(projectRoot, opts);
+  } catch {
+    return { hits: [], search: 'missing-wiki' };
+  }
+
+  const docs = pages.map((page) => ({
+    id: page.rel,
+    title: page.title,
+    path: page.rel,
+    body: page.body,
+    type: page.type || '',
+    memoryType: page.memoryType || '',
+  }));
+  const MiniSearch = await loadMiniSearch();
+  const byPath = pageLookup(pages);
+  const hits = new Map();
+  let search = 'substring+wikilink';
+
+  if (MiniSearch) {
+    search = 'minisearch+wikilink';
+    const index = new MiniSearch({
+      fields: ['title', 'path', 'body'],
+      storeFields: ['title', 'path', 'type', 'memoryType'],
+      searchOptions: {
+        boost: { title: 3, path: 2 },
+        prefix: true,
+      },
+    });
+    index.addAll(docs);
+
+    for (const item of index.search(query, { prefix: true, boost: { title: 3, path: 2 } })) {
+      const page = byPath.get(item.id);
+      if (!page) continue;
+      const subScore = substringScore(page, terms);
+      const score = item.score + subScore;
+      hits.set(page.rel, resultRecord(page, score, {
+        directScore: score,
+        matchedTerms: item.terms || terms,
+        snippet: snippetFor(page, terms),
+      }));
+    }
+  }
+
+  for (const page of pages) {
+    const score = substringScore(page, terms);
+    if (score <= 0 || hits.has(page.rel)) continue;
+    hits.set(page.rel, resultRecord(page, score, {
+      directScore: score,
+      matchedTerms: terms,
+      snippet: snippetFor(page, terms),
+    }));
+  }
+
+  if (expand && hits.size > 0) {
+    const graph = buildWikiGraph(pages);
+    const direct = [...hits.values()].sort(sortHits).slice(0, Math.max(limit, 5));
+    for (const seed of direct) {
+      const neighbors = new Set([
+        ...(graph.outlinks.get(seed.path) || []),
+        ...(graph.backlinks.get(seed.path) || []),
+      ]);
+      for (const neighborPath of neighbors) {
+        if (hits.has(neighborPath)) continue;
+        const page = byPath.get(neighborPath);
+        if (!page) continue;
+        const linkScore = seed.score * 0.2;
+        hits.set(neighborPath, resultRecord(page, linkScore, {
+          linkScore,
+          source: 'linked',
+          via: [seed.path],
+          matchedTerms: [],
+          snippet: snippetFor(page, terms),
+        }));
+      }
+    }
+  }
+
+  return {
+    hits: [...hits.values()].sort(sortHits).slice(0, limit),
+    search,
+  };
+}
+
+function redactHit(hit) {
+  return {
+    ...hit,
+    path: redactText(hit.path, 300),
+    title: redactText(hit.title, 300),
+    via: Array.isArray(hit.via) ? hit.via.map((item) => redactText(item, 300)) : [],
+    matchedTerms: Array.isArray(hit.matchedTerms) ? hit.matchedTerms.map((item) => redactText(item, 120)) : [],
+    snippet: redactText(hit.snippet, SNIPPET_CHARS),
+  };
+}
+
+export async function buildContextPack(projectRoot, query, options = {}) {
+  const limit = Number(options.limit || DEFAULT_LIMIT);
+  const expand = options.expand !== false;
+  const memoryExcerpt = await readMemoryExcerpt(projectRoot);
+  const indexExcerpt = (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'index.md'))).slice(0, 1200).trim();
+  const logExcerpt = options.includeLog
+    ? (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'log.md'))).slice(-1000).trim()
+    : '';
+  const result = query ? await performSearch(projectRoot, query, { ...options, limit, expand }) : { hits: [], search: 'none' };
+  return {
+    workspace: projectRoot,
+    query: redactText(query || '', 1000),
+    limit,
+    expand,
+    search: result.search,
+    memoryExcerpt: redactText(memoryExcerpt.trim(), 30000),
+    indexExcerpt: redactText(indexExcerpt, 2000),
+    logExcerpt: redactText(logExcerpt, 2000),
+    hits: result.hits.map(redactHit),
+  };
+}
+
+export function formatContextPack(pack) {
+  const lines = [
+    'LLM Wiki context from llm-wiki-kit:',
+    '- Treat chat memory as temporary; update project Markdown when knowledge should persist.',
+    '- Preserve raw/wiki separation. Do not store secrets, tokens, .env contents, private keys, or personal/customer identifiers.',
+    '- Prefer updating existing wiki pages over creating duplicate pages.',
+  ];
+  if (pack.query) {
+    lines.push(`- query: "${pack.query}"`);
+    lines.push(`- search: ${pack.search}`);
+  }
+  if (pack.memoryExcerpt) {
+    lines.push('', 'Memory excerpt:', pack.memoryExcerpt);
+  }
+  if (pack.indexExcerpt) {
+    lines.push('', 'Index excerpt:', pack.indexExcerpt);
+  }
+  if (pack.hits.length > 0) {
+    lines.push('', 'Relevant wiki pages:');
+    for (const hit of pack.hits) {
+      const suffix = hit.source === 'linked' && hit.via.length > 0
+        ? `, linked via ${hit.via.join(', ')}`
+        : '';
+      lines.push(`- ${hit.path} (score ${hit.score.toFixed(2)}, ${hit.source}${suffix}): ${hit.snippet}`);
+    }
+  }
+  if (pack.logExcerpt) {
+    lines.push('', 'Recent log excerpt:', pack.logExcerpt);
+  }
+  return lines.join('\n').trim();
+}