@zosmaai/pi-llm-wiki 0.7.2 → 0.7.4

package/extensions/llm-wiki/lib/recall.ts

@@ -7,6 +7,7 @@ import {
   type VaultPaths,
   getPersonalWikiPaths,
   isPersonalVault,
+  parseFrontmatter,
   readJson,
   resolveVaultPaths,
 } from "./utils.js";
@@ -26,64 +27,415 @@ export interface RecallResult {
   path: string;
   /** Vault source label for dual-vault results */
   vaultLabel?: string;
+  /** Relevance score (higher = better match). Used for filtering auto-injected results. */
+  score: number;
+}
+
+type Scored = {
+  id: string;
+  entry: Registry["pages"][string];
+  score: number;
+  pagePath: string;
+  bestChunkPreview: string;
+};
+
+/**
+ * Normalize text for recall matching.
+ *
+ * Wiki queries are often short and multilingual (for example: "继续学习pi").
+ * Normalization keeps CJK characters intact, lowercases Latin text, removes
+ * punctuation boundaries, and makes hyphenated page IDs match space-separated
+ * queries.
+ */
+function normalizeText(value: unknown): string {
+  return flattenSearchValue(value)
+    .toLowerCase()
+    .normalize("NFKC")
+    .replace(/[\-_./\\]+/g, " ")
+    .replace(/[\p{P}\p{S}]+/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function compactText(value: string): string {
+  return value.replace(/\s+/g, "");
+}
+
+function flattenSearchValue(value: unknown): string {
+  if (value == null) return "";
+  if (Array.isArray(value)) return value.map(flattenSearchValue).join(" ");
+  if (typeof value === "object") return Object.values(value).map(flattenSearchValue).join(" ");
+  return String(value);
+}
+
+function unique(values: string[]): string[] {
+  return [...new Set(values.filter(Boolean))];
+}
+
+/**
+ * Tokenize with support for CJK short queries and English/kebab-case terms.
+ *
+ * Besides whitespace tokens, this returns Latin/digit runs ("pi", "recall")
+ * and overlapping CJK bigrams/trigrams. The full normalized query is also kept
+ * so exact short phrases still rank highest.
+ */
+function queryTerms(query: string): string[] {
+  const normalized = normalizeText(query);
+  const compact = compactText(normalized);
+  const terms: string[] = [];
+
+  if (normalized) terms.push(normalized);
+  if (compact && compact !== normalized) terms.push(compact);
+
+  for (const part of normalized.split(/\s+/)) {
+    if (part.length >= 2) terms.push(part);
+  }
+
+  const latinRuns = normalized.match(/[a-z0-9]{2,}/g) ?? [];
+  terms.push(...latinRuns);
+
+  const cjkRuns =
+    normalized.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+/gu) ?? [];
+  for (const run of cjkRuns) {
+    for (let size = 2; size <= 3; size++) {
+      if (run.length < size) continue;
+      for (let i = 0; i <= run.length - size; i++) {
+        terms.push(run.slice(i, i + size));
+      }
+    }
+  }
+
+  return unique(terms).slice(0, 30);
+}
+
+function includesTerm(haystack: string, term: string): boolean {
+  if (!haystack || !term) return false;
+  return haystack.includes(term) || compactText(haystack).includes(compactText(term));
+}
+
+function scoreField(value: unknown, terms: string[], weight: number): number {
+  const text = normalizeText(value);
+  if (!text) return 0;
+
+  let score = 0;
+  for (const term of terms) {
+    if (includesTerm(text, term)) score += weight;
+  }
+  return score;
+}
+
+// ─── Common English stopwords ─────────────────────────
+
+const STOPWORDS = new Set([
+  "the",
+  "this",
+  "that",
+  "with",
+  "from",
+  "have",
+  "been",
+  "were",
+  "they",
+  "their",
+  "them",
+  "will",
+  "would",
+  "could",
+  "should",
+  "about",
+  "there",
+  "which",
+  "what",
+  "when",
+  "where",
+  "than",
+  "then",
+  "also",
+  "just",
+  "more",
+  "some",
+  "such",
+  "only",
+  "other",
+  "into",
+  "over",
+  "very",
+  "after",
+  "before",
+  "because",
+  "between",
+  "through",
+  "during",
+  "without",
+  "within",
+  "along",
+  "these",
+  "those",
+  "page",
+  "section",
+  "note",
+  "info",
+  "type",
+  "used",
+  "using",
+]);
+
+// ─── Chunk-Level Indexing ────────────────────────────
+
+interface PageChunk {
+  /** The heading line (e.g. "## Configuration") or empty for the intro section */
+  heading: string;
+  /** Content of this chunk */
+  content: string;
+  /** Heading level (0 for intro, 1 for #, 2 for ##, etc.) */
+  level: number;
+}
+
+/**
+ * Split a page's body into chunks by headings.
+ * Each heading and its following content become one chunk.
+ * Content before the first heading becomes the intro chunk.
+ */
+function chunkPage(body: string): PageChunk[] {
+  if (!body.trim()) return [];
+
+  const chunks: PageChunk[] = [];
+  const lines = body.split("\n");
+
+  let currentHeading = "";
+  let currentLevel = 0;
+  let currentContent: string[] = [];
+
+  for (const line of lines) {
+    const headingMatch = line.trim().match(/^(#{1,6})\s+(.+)$/);
+    if (headingMatch) {
+      // Save previous chunk
+      if (currentContent.length > 0 || currentHeading) {
+        chunks.push({
+          heading: currentHeading,
+          content: currentContent.join("\n").trim(),
+          level: currentLevel,
+        });
+      }
+      currentHeading = headingMatch[2].trim();
+      currentLevel = headingMatch[1].length;
+      currentContent = [];
+    } else {
+      currentContent.push(line);
+    }
+  }
+
+  // Save last chunk
+  if (currentContent.length > 0 || currentHeading) {
+    chunks.push({
+      heading: currentHeading,
+      content: currentContent.join("\n").trim(),
+      level: currentLevel,
+    });
+  }
+
+  return chunks;
+}
+
+function pagePreview(content: string): string {
+  const { body } = parseFrontmatter(content);
+  return body.trim().slice(0, 200).replace(/\n/g, " ");
+}
+
+/**
+ * Get a preview of the best-matching chunk, or fall back to the page intro.
+ * Shows the heading (if any) and the first ~200 chars of content.
+ */
+function chunkPreview(heading: string, content: string): string {
+  const trimmed = content.slice(0, 180).replace(/\n/g, " ");
+  if (heading) {
+    return `#${heading} — ${trimmed}`;
+  }
+  return trimmed;
+}
+
+/**
+ * Extract distinctive terms from the top search results for query expansion.
+ * Pseudo-relevance feedback: terms from top-matching pages that aren't in
+ * the original query become expansion candidates.
+ */
+function extractExpansionTerms(
+  scored: Scored[],
+  originalQuery: string,
+  paths: VaultPaths,
+  maxTerms = 6,
+): string[] {
+  const topResults = scored.slice(0, Math.min(3, scored.length));
+  if (topResults.length === 0) return [];
+
+  const originalNorm = normalizeText(originalQuery);
+  const termFreq = new Map<string, number>();
+
+  for (const { pagePath, entry } of topResults) {
+    // Collect text from registry metadata + file content
+    const metaText = normalizeText(
+      [entry.title, entry.aliases, entry.tags, entry.summary, entry.description]
+        .filter(Boolean)
+        .join(" "),
+    );
+    for (const w of metaText.split(/\s+/)) {
+      if (w.length >= 4 && !originalNorm.includes(w) && !STOPWORDS.has(w)) {
+        termFreq.set(w, (termFreq.get(w) || 0) + 1);
+      }
+    }
+
+    // Also extract from file body
+    if (existsSync(pagePath)) {
+      const content = readFileSync(pagePath, "utf-8");
+      const { body } = parseFrontmatter(content);
+      const bodyNorm = normalizeText(body);
+      for (const w of bodyNorm.split(/\s+/)) {
+        if (w.length >= 4 && !originalNorm.includes(w) && !STOPWORDS.has(w)) {
+          termFreq.set(w, (termFreq.get(w) || 0) + 1);
+        }
+      }
+    }
+  }
+
+  // Sort by frequency descending, take top N
+  return Array.from(termFreq.entries())
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .slice(0, maxTerms)
+    .map(([term]) => term);
 }
 
 /**
  * Search a single vault's registry for pages matching a query.
  * Returns up to `maxResults` matches, each with a content preview.
+ * Results below `minScore` are excluded (default 0 = no filtering).
  */
-export function searchWiki(paths: VaultPaths, query: string, maxResults = 5): RecallResult[] {
+export function searchWiki(
+  paths: VaultPaths,
+  query: string,
+  maxResults = 5,
+  minScore = 0,
+): RecallResult[] {
   const registry = readJson<Registry>(join(paths.meta, "registry.json"), {
     version: "1.0",
     last_updated: "",
     pages: {},
   });
 
-  const q = query.toLowerCase();
-  const terms = q
-    .split(/\s+/)
-    .filter((t) => t.length > 2)
-    .slice(0, 10);
-
+  const terms = queryTerms(query);
   if (terms.length === 0) return [];
 
-  type Scored = { id: string; entry: Registry["pages"][string]; score: number };
   const scored: Scored[] = [];
 
   for (const [id, entry] of Object.entries(registry.pages)) {
+    const pagePath = join(paths.wiki, `${id}.md`);
+    const content = existsSync(pagePath) ? readFileSync(pagePath, "utf-8") : "";
+    const { frontmatter, body } = parseFrontmatter(content);
+
     let score = 0;
-    const title = String(entry.title || "").toLowerCase();
-    const type = String(entry.type || "").toLowerCase();
 
-    for (const term of terms) {
-      if (id.toLowerCase().includes(term)) score += 3;
-      if (title.includes(term)) score += 4;
-      if (type.includes(term)) score += 1;
+    // Strong identifiers: exact command/short-query aliases should win.
+    score += scoreField(id, terms, 3);
+    score += scoreField(entry.title, terms, 5);
+    score += scoreField(frontmatter.title, terms, 5);
+    score += scoreField(entry.type, terms, 1);
+
+    // Recall-oriented metadata. Arrays are supported by parseFrontmatter and
+    // legacy comma/bracket strings still flatten into searchable text.
+    score += scoreField(entry.aliases, terms, 6);
+    score += scoreField(frontmatter.aliases, terms, 6);
+    score += scoreField(entry.recall_triggers, terms, 7);
+    score += scoreField(frontmatter.recall_triggers, terms, 7);
+    score += scoreField(entry.summary, terms, 3);
+    score += scoreField(frontmatter.summary, terms, 3);
+    score += scoreField(entry.description, terms, 3);
+    score += scoreField(frontmatter.description, terms, 3);
+
+    // General metadata from the registry/frontmatter.
+    score += scoreField(entry.tags, terms, 2);
+    score += scoreField(entry.category, terms, 2);
+    score += scoreField(entry.domain, terms, 2);
+    score += scoreField(frontmatter.tags, terms, 2);
+    score += scoreField(frontmatter.category, terms, 2);
+    score += scoreField(frontmatter.domain, terms, 2);
+
+    // Body search: use chunk-level indexing for more precise matching.
+    // Each section of the page is scored independently, so a query about
+    // "Postgres" matches only the Postgres section, not the whole page.
+    let bestChunkScore = 0;
+    let bestChunkHeading = "";
+    let bestChunkContent = "";
+
+    if (body.trim()) {
+      const chunks = chunkPage(body);
+      for (const chunk of chunks) {
+        let chunkScore = 0;
+        // Heading gets a strong boost
+        chunkScore += scoreField(chunk.heading, terms, 4);
+        // Chunk body content
+        chunkScore += scoreField(chunk.content, terms, 1);
+
+        if (chunkScore > bestChunkScore) {
+          bestChunkScore = chunkScore;
+          bestChunkHeading = chunk.heading;
+          bestChunkContent = chunk.content;
+        }
+      }
     }
 
-    // Boost if query terms appear in tags/categories
-    const tags = String(entry.tags || entry.category || entry.domain || "");
-    for (const term of terms) {
-      if (tags.toLowerCase().includes(term)) score += 2;
-    }
+    // Add best chunk score to total page score
+    score += bestChunkScore;
 
     if (score > 0) {
-      scored.push({ id, entry, score });
+      scored.push({
+        id,
+        entry,
+        score,
+        pagePath,
+        bestChunkPreview: bestChunkContent ? chunkPreview(bestChunkHeading, bestChunkContent) : "",
+      });
     }
   }
 
-  scored.sort((a, b) => b.score - a.score);
-  const top = scored.slice(0, maxResults);
+  scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+
+  // ── Pseudo-Relevance Feedback (PRF) ─────────────────
+  // Extract distinctive terms from the top 3 results and use them to
+  // boost semantically related pages. This gives "semantic" expansion
+  // without external dependencies: if an "Authentication" page mentions
+  // JWT, OAuth, and sessions, those terms boost other pages that discuss
+  // related concepts.
+  const expansionTerms = extractExpansionTerms(scored, query, paths, 6);
+  if (expansionTerms.length > 0) {
+    const expTermList = queryTerms(expansionTerms.join(" "));
+    // Apply expansion scoring to the top 25 results (cheap re-read)
+    const expansionCandidates = scored.slice(0, Math.min(25, scored.length));
+    for (const item of expansionCandidates) {
+      const content = existsSync(item.pagePath) ? readFileSync(item.pagePath, "utf-8") : "";
+      const { body } = parseFrontmatter(content);
+      let expChunkScore = 0;
+      if (body.trim()) {
+        const chunks = chunkPage(body);
+        for (const chunk of chunks) {
+          let cs = 0;
+          cs += scoreField(chunk.heading, expTermList, 2); // half weight
+          cs += scoreField(chunk.content, expTermList, 0.5);
+          if (cs > expChunkScore) expChunkScore = cs;
+        }
+      }
+      // Dampened addition — expansion contributes at most 40%
+      item.score += expChunkScore * 0.4;
+    }
+  }
 
-  return top.map(({ id, entry }) => {
-    // Try to read page content for preview
-    let preview = "";
-    const pagePath = join(paths.wiki, `${id}.md`);
-    if (existsSync(pagePath)) {
-      const content = readFileSync(pagePath, "utf-8");
-      // Strip frontmatter
-      const body = content.replace(/^---[\s\S]*?---\n/, "").trim();
-      preview = body.slice(0, 200).replace(/\n/g, " ");
+  // Re-sort after expansion scoring
+  scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+  const top = scored.filter((s) => s.score >= minScore).slice(0, maxResults);
+
+  return top.map(({ id, entry, pagePath, score, bestChunkPreview }) => {
+    let preview = bestChunkPreview;
+    if (!preview && existsSync(pagePath)) {
+      // Fallback: no chunk matched, show page intro
+      preview = pagePreview(readFileSync(pagePath, "utf-8"));
     }
 
     return {
@@ -92,33 +444,40 @@ export function searchWiki(paths: VaultPaths, query: string, maxResults = 5): Re
       type: String(entry.type || "page"),
       preview,
       path: pagePath,
+      score,
     };
   });
 }
 
-/**
- * Format recall results as a compact system-prompt section.
- */
 /**
  * Search both project/primary vault and personal vault, merging results.
  * Personal results are appended after primary results, deduplicated by page ID.
+ *
+ * @param minScore - Minimum relevance score (default 0 = no filter).
+ * @param includePersonal - Whether to search the personal vault (default true).
+ *   Auto-injection should pass false to avoid personal-vault contamination.
  */
 export function searchWikiLayered(
   primaryPaths: VaultPaths,
   query: string,
   maxResults = 5,
+  minScore = 0,
+  includePersonal = true,
 ): RecallResult[] {
   // Search primary vault
-  const primaryResults = searchWiki(primaryPaths, query, maxResults);
+  const primaryResults = searchWiki(primaryPaths, query, maxResults, minScore);
 
   // If primary is already the personal vault, no layered search needed
   if (isPersonalVault(primaryPaths)) return primaryResults;
 
-  // Search personal vault as secondary layer
-  const personalPaths = getPersonalWikiPaths();
-  if (!existsSync(join(personalPaths.dotWiki, "config.json"))) return primaryResults;
-
-  const personalResults = searchWiki(personalPaths, query, maxResults);
+  // Search personal vault as secondary layer (only when explicitly requested)
+  let personalResults: RecallResult[] = [];
+  if (includePersonal) {
+    const personalPaths = getPersonalWikiPaths();
+    if (existsSync(join(personalPaths.dotWiki, "config.json"))) {
+      personalResults = searchWiki(personalPaths, query, maxResults, minScore);
+    }
+  }
 
   // Merge: personal results first (they're the user's accumulated knowledge),
   // then primary results (project-specific). Deduplicate by page ID.
@@ -241,25 +600,15 @@ export function registerWikiRecall(pi: ExtensionAPI): void {
         content: [
           {
             type: "text",
-            text: [
-              `🧠 **${results.length} wiki page(s) relevant** to "${params.query}"${layerTag}:`,
-              "",
-              ...results.map(
-                (r) =>
-                  `- ${r.vaultLabel || "📁"} [[${r.id}]] — *${r.type}* — ${r.title}${
-                    r.preview ? `\n  > ${r.preview.slice(0, 150)}` : ""
-                  }`,
-              ),
-              "",
-              "Use `read` on any page for full content.",
-              "Use `wiki_retro` to save new insights from this task.",
-            ].join("\n"),
+            text: `Found ${results.length} wiki page(s) matching "${params.query}"${layerTag}:\n\n${results
+              .map((r) => {
+                const vault = r.vaultLabel ? ` ${r.vaultLabel}` : "";
+                return `## [[${r.id}]] — ${r.title}${vault}\nType: ${r.type}\nPath: ${r.path}\n\n${r.preview}`;
+              })
+              .join("\n\n---\n\n")}`,
           },
         ],
-        details: { query: params.query, matches: results.map((r) => r.id) } as Record<
-          string,
-          unknown
-        >,
+        details: { query: params.query, matches: results } as Record<string, unknown>,
       };
     },
   });

package/extensions/llm-wiki/lib/utils.ts

@@ -184,6 +184,22 @@ export function nextSourceId(paths: VaultPaths): string {
   return `${prefix}-${String(num + 1).padStart(3, "0")}`;
 }
 
+/** Parse a small, dependency-free YAML scalar/inline-array value. */
+function parseFrontmatterValue(raw: string, unquote = false): unknown {
+  const trimmed = raw.trim();
+  const unquoted = (value: string) => value.replace(/^(["'])(.*)\1$/, "$2").trim();
+
+  if (!trimmed) return "";
+
+  if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
+    const inner = trimmed.slice(1, -1).trim();
+    if (!inner) return [];
+    return inner.split(",").map((item) => unquoted(item.trim()));
+  }
+
+  return unquote ? unquoted(trimmed) : trimmed;
+}
+
 /** Extract frontmatter from markdown. */
 export function parseFrontmatter(content: string): {
   frontmatter: Record<string, unknown>;
@@ -194,12 +210,33 @@ export function parseFrontmatter(content: string): {
 
   const frontmatter: Record<string, unknown> = {};
   const lines = match[1].split("\n");
+  let currentListKey: string | null = null;
+
   for (const line of lines) {
+    const listMatch = line.match(/^\s*-\s+(.*)$/);
+    if (listMatch && currentListKey) {
+      const current = frontmatter[currentListKey];
+      const list = Array.isArray(current) ? current : [];
+      list.push(parseFrontmatterValue(listMatch[1], true));
+      frontmatter[currentListKey] = list;
+      continue;
+    }
+
     const idx = line.indexOf(":");
-    if (idx > 0) {
-      const key = line.slice(0, idx).trim();
-      const val = line.slice(idx + 1).trim();
-      frontmatter[key] = val;
+    if (idx <= 0) {
+      currentListKey = null;
+      continue;
+    }
+
+    const key = line.slice(0, idx).trim();
+    const val = line.slice(idx + 1).trim();
+
+    if (!val) {
+      frontmatter[key] = [];
+      currentListKey = key;
+    } else {
+      frontmatter[key] = parseFrontmatterValue(val);
+      currentListKey = null;
     }
   }
   return { frontmatter, body: match[2] };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
   "keywords": [
     "pi",