@zosmaai/pi-llm-wiki 0.10.7 → 0.11.0

package/extensions/llm-wiki/lib/observation.ts

@@ -1,11 +1,12 @@
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
 import { Type } from "typebox";
 import { scheduleReindex } from "./indexing.js";
+import { createKnowledgeDocument, writeKnowledgeDocumentFile } from "./knowledge-document.js";
 import { appendEvent, rebuildMetadataLight } from "./metadata.js";
 import type { Runtime } from "./runtime.js";
 import { type VaultPaths, fmtDate, resolveVaultPaths } from "./utils.js";
+import { assertWritableVault, inspectWritableVault } from "./vault-format.js";
 
 // ─── Types ─────────────────────────────────────────────
 
@@ -51,6 +52,7 @@ export function saveObservation(
   input: ObservationInput,
   opts?: { rebuild?: boolean },
 ): ObservationResult {
+  assertWritableVault(paths);
   const today = fmtDate();
   const timestamp = new Date().toISOString();
 
@@ -62,50 +64,38 @@ export function saveObservation(
     .slice(0, 60);
   const slug = `obs-${today}-${slugBase}`;
 
-  // Write to wiki/sources/{slug}.md
-  const sourcePageDir = join(paths.wiki, "sources");
-  mkdirSync(sourcePageDir, { recursive: true });
-  const pagePath = join(sourcePageDir, `${slug}.md`);
+  const pagePath = join(paths.wiki, "sources", `${slug}.md`);
 
   const relevanceEmoji = RELEVANCE_EMOJIS[input.relevance] ?? "📝";
   const tags = input.tags ?? "";
   const sourceContext = input.source_context ?? "";
 
-  const pageContent = [
-    "---",
-    "type: source",
-    `title: "Observation: ${input.title}"`,
-    `slug: ${slug}`,
-    "status: observation",
-    `created: ${today}`,
-    `updated: ${today}`,
-    `relevance: ${input.relevance}`,
-    `observed_at: ${timestamp}`,
-    tags
-      ? `tags: [${tags
-          .split(/\s+/)
-          .filter(Boolean)
-          .map((t) => `"${t}"`)
-          .join(", ")}]`
-      : "",
-    sourceContext ? `source_context: "${sourceContext}"` : "",
-    "---",
-    "",
-    `# ${relevanceEmoji} Observation: ${input.title}`,
-    "",
-    input.content,
-    "",
-    `*Relevance: ${input.relevance}*`,
-    sourceContext ? `\n*Context: ${sourceContext}*` : "",
-    tags ? `\n*Tags: ${tags}*` : "",
-    "",
-    "---",
-    `*Observed: ${timestamp}*`,
-    "",
-  ]
-    .filter((l) => l !== "")
-    .join("\n");
-  writeFileSync(pagePath, pageContent, "utf-8");
+  const body = `# ${relevanceEmoji} Observation: ${input.title}
+
+${input.content}
+
+*Relevance: ${input.relevance}*${sourceContext ? `\n*Context: ${sourceContext}*` : ""}${tags ? `\n*Tags: ${tags}*` : ""}
+
+---
+*Observed: ${timestamp}*`;
+
+  const doc = createKnowledgeDocument(
+    `sources/${slug}.md`,
+    {
+      type: "source",
+      title: `Observation: ${input.title}`,
+      slug,
+      status: "observation",
+      created: today,
+      updated: today,
+      relevance: input.relevance,
+      observed_at: timestamp,
+      ...(tags ? { tags: tags.split(/\s+/).filter(Boolean) } : {}),
+      ...(sourceContext ? { source_context: sourceContext } : {}),
+    },
+    body,
+  );
+  writeKnowledgeDocumentFile(pagePath, doc);
 
   // Log event
   appendEvent(paths, {
@@ -213,15 +203,19 @@ export function registerWikiObserve(
     async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
       const paths = resolveVaultPaths(ctx.cwd ?? process.cwd());
 
-      if (!existsSync(join(paths.dotWiki, "config.json"))) {
+      const vaultCheck = inspectWritableVault(paths);
+      if (!vaultCheck.ok) {
         return {
           content: [
             {
               type: "text",
-              text: "No wiki vault found at this location. Initialize one with wiki_bootstrap first.",
+              text: `Wiki vault error: ${vaultCheck.diagnostics[0].message}`,
             },
           ],
-          details: { error: "no_vault" } as Record<string, unknown>,
+          details: {
+            error: vaultCheck.diagnostics[0].code,
+            diagnostics: vaultCheck.diagnostics,
+          } as Record<string, unknown>,
           isError: true,
         };
       }

package/extensions/llm-wiki/lib/recall.ts

@@ -10,6 +10,7 @@ import {
   readEmbeddingStore,
   resolveEmbedder,
 } from "./embeddings.js";
+import { parseKnowledgeDocument } from "./knowledge-document.js";
 import type { Registry } from "./metadata.js";
 import type { Runtime } from "./runtime.js";
 import type { TaskConfig } from "./task-config.js";
@@ -17,10 +18,10 @@ import {
   type VaultPaths,
   getPersonalWikiPaths,
   isPersonalVault,
-  parseFrontmatter,
   readJson,
   resolveVaultPaths,
 } from "./utils.js";
+import { inspectVaultFormat } from "./vault-format.js";
 
 // ─── Public API ────────────────────────────────────────
 
@@ -296,8 +297,19 @@ function chunkPage(body: string): PageChunk[] {
   return chunks;
 }
 
+function parsePage(
+  path: string,
+  id: string,
+): { frontmatter: Record<string, unknown>; body: string } | undefined {
+  if (!existsSync(path)) return undefined;
+  const content = readFileSync(path, "utf-8");
+  const result = parseKnowledgeDocument(content, `${id}.md`);
+  if (!result.ok) return undefined;
+  return { frontmatter: result.document.frontmatter, body: result.document.body };
+}
+
 function pagePreview(content: string): string {
-  const { body } = parseFrontmatter(content);
+  const body = content.replace(/^---\n[\s\S]*?\n---\n?/, "");
   return body.trim().slice(0, 200).replace(/\n/g, " ");
 }
 
@@ -330,7 +342,7 @@ function extractExpansionTerms(
   const originalNorm = normalizeText(originalQuery);
   const termFreq = new Map<string, number>();
 
-  for (const { pagePath, entry } of topResults) {
+  for (const { id, pagePath, entry } of topResults) {
     // Collect text from registry metadata + file content
     const metaText = normalizeText(
       [entry.title, entry.aliases, entry.tags, entry.summary, entry.description]
@@ -345,12 +357,13 @@ function extractExpansionTerms(
 
     // Also extract from file body
     if (existsSync(pagePath)) {
-      const content = readFileSync(pagePath, "utf-8");
-      const { body } = parseFrontmatter(content);
-      const bodyNorm = normalizeText(body);
-      for (const w of bodyNorm.split(/\s+/)) {
-        if (w.length >= 4 && !originalNorm.includes(w) && !STOPWORDS.has(w)) {
-          termFreq.set(w, (termFreq.get(w) || 0) + 1);
+      const parsed = parsePage(pagePath, id);
+      if (parsed) {
+        const bodyNorm = normalizeText(parsed.body);
+        for (const w of bodyNorm.split(/\s+/)) {
+          if (w.length >= 4 && !originalNorm.includes(w) && !STOPWORDS.has(w)) {
+            termFreq.set(w, (termFreq.get(w) || 0) + 1);
+          }
         }
       }
     }
@@ -394,8 +407,11 @@ export function searchWiki(
 
   for (const [id, entry] of Object.entries(registry.pages)) {
     const pagePath = join(paths.wiki, `${id}.md`);
-    const content = existsSync(pagePath) ? readFileSync(pagePath, "utf-8") : "";
-    const { frontmatter, body } = parseFrontmatter(content);
+    const pageExists = existsSync(pagePath);
+    const parsed = parsePage(pagePath, id);
+    if (pageExists && !parsed) continue;
+    const frontmatter = parsed?.frontmatter ?? {};
+    const body = parsed?.body ?? "";
 
     let score = 0;
 
@@ -405,8 +421,7 @@ export function searchWiki(
     score += scoreField(frontmatter.title, terms, 5);
     score += scoreField(entry.type, terms, 1);
 
-    // Recall-oriented metadata. Arrays are supported by parseFrontmatter and
-    // legacy comma/bracket strings still flatten into searchable text.
+    // Recall-oriented metadata.
     score += scoreField(entry.aliases, terms, 6);
     score += scoreField(frontmatter.aliases, terms, 6);
     score += scoreField(entry.recall_triggers, terms, 7);
@@ -489,8 +504,8 @@ export function searchWiki(
     // Apply expansion scoring to the top 25 results (cheap re-read)
     const expansionCandidates = scored.slice(0, Math.min(25, scored.length));
     for (const item of expansionCandidates) {
-      const content = existsSync(item.pagePath) ? readFileSync(item.pagePath, "utf-8") : "";
-      const { body } = parseFrontmatter(content);
+      const parsed = parsePage(item.pagePath, item.id);
+      const body = parsed?.body ?? "";
       let expChunkScore = 0;
       if (body.trim()) {
         const chunks = chunkPage(body);
@@ -960,6 +975,10 @@ export function registerWikiRecall(pi: ExtensionAPI, runtime?: Runtime): void {
       }
 
       const maxResults = Math.min(params.max_results ?? 5, 10);
+      const vaultDiagnostics = inspectVaultFormat(paths).diagnostics;
+      const diagnosticText = vaultDiagnostics.length
+        ? `\n\nDiagnostics: ${vaultDiagnostics.map((diagnostic) => diagnostic.code).join(", ")}`
+        : "";
       // Use layered hybrid search: personal vault + project vault, blending
       // lexical scoring with precomputed semantic embeddings when available.
       // No embeddings / no embedder => pure lexical, no network call.
@@ -973,10 +992,14 @@ export function registerWikiRecall(pi: ExtensionAPI, runtime?: Runtime): void {
           content: [
             {
               type: "text",
-              text: `No wiki pages found matching "${params.query}". The wiki is empty — use wiki_retro to start building knowledge.`,
+              text: `No wiki pages found matching "${params.query}". The wiki is empty — use wiki_retro to start building knowledge.${diagnosticText}`,
             },
           ],
-          details: { query: params.query, matches: [] } as Record<string, unknown>,
+          details: {
+            query: params.query,
+            matches: [],
+            diagnostics: vaultDiagnostics,
+          } as Record<string, unknown>,
         };
       }
 
@@ -997,19 +1020,22 @@ export function registerWikiRecall(pi: ExtensionAPI, runtime?: Runtime): void {
             return `${i + 1}. [[${r.id}]] — ${r.title} (${r.type}, score ${r.score.toFixed(1)})${vault}\n   Path: ${r.path}${tail}`;
           })
           .join("\n");
-        const text = [
-          `Found ${results.length} wiki page(s) matching "${params.query}"${layerTag} (two-stage recall — ranked links, expand on demand):`,
-          "",
-          linkLines,
-          "",
-          "Call `read` on the path(s) you need to pull full content.",
-        ].join("\n");
+        const text =
+          [
+            `Found ${results.length} wiki page(s) matching "${params.query}"${layerTag} (two-stage recall — ranked links, expand on demand):`,
+            "",
+            linkLines,
+            "",
+            "Call `read` on the path(s) you need to pull full content.",
+          ].join("\n") + diagnosticText;
         return {
           content: [{ type: "text", text }],
-          details: { query: params.query, mode: "links", matches: results } as Record<
-            string,
-            unknown
-          >,
+          details: {
+            query: params.query,
+            mode: "links",
+            matches: results,
+            diagnostics: vaultDiagnostics,
+          } as Record<string, unknown>,
         };
       }
 
@@ -1022,13 +1048,15 @@ export function registerWikiRecall(pi: ExtensionAPI, runtime?: Runtime): void {
                 const vault = r.vaultLabel ? ` ${r.vaultLabel}` : "";
                 return `## [[${r.id}]] — ${r.title}${vault}\nType: ${r.type}\nPath: ${r.path}\n\n${r.preview}`;
               })
-              .join("\n\n---\n\n")}`,
+              .join("\n\n---\n\n")}${diagnosticText}`,
           },
         ],
-        details: { query: params.query, mode: "preview", matches: results } as Record<
-          string,
-          unknown
-        >,
+        details: {
+          query: params.query,
+          mode: "preview",
+          matches: results,
+          diagnostics: vaultDiagnostics,
+        } as Record<string, unknown>,
       };
     },
   });

package/extensions/llm-wiki/lib/retro.ts

@@ -1,11 +1,12 @@
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import { dirname, resolve } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { Type } from "typebox";
 import { scheduleReindex } from "./indexing.js";
+import { createKnowledgeDocument, writeKnowledgeDocumentFile } from "./knowledge-document.js";
 import { appendEvent, rebuildMetadataLight } from "./metadata.js";
 import type { Runtime } from "./runtime.js";
 import { type VaultPaths, fmtDate, resolveVaultPaths } from "./utils.js";
+import { assertWritableVault, inspectWritableVault } from "./vault-format.js";
 
 // ─── Public API ────────────────────────────────────────
 
@@ -24,6 +25,16 @@ export interface RetroResult {
  * The 4-layer pipeline (raw → source pages → canonical pages → metadata)
  * is still available via wiki_capture_source → wiki_ingest for deep research.
  */
+function insightPath(paths: VaultPaths, slug: string): string {
+  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug) || slug === "index" || slug === "log") {
+    throw new Error(`Invalid insight slug: ${slug}`);
+  }
+  const directory = resolve(paths.wiki, "sources");
+  const target = resolve(directory, `${slug}.md`);
+  if (dirname(target) !== directory) throw new Error(`Invalid insight slug: ${slug}`);
+  return target;
+}
+
 export function saveInsight(
   paths: VaultPaths,
   slug: string,
@@ -32,41 +43,38 @@ export function saveInsight(
   category?: string,
   opts?: { rebuild?: boolean },
 ): RetroResult {
+  assertWritableVault(paths);
   const today = fmtDate();
 
-  // Write a single markdown file to wiki/sources/{slug}.md
-  const sourcePageDir = join(paths.wiki, "sources");
-  mkdirSync(sourcePageDir, { recursive: true });
-  const sourcePagePath = join(sourcePageDir, `${slug}.md`);
-
-  const pageContent = [
-    "---",
-    "type: source",
-    `title: "${title}"`,
-    `slug: ${slug}`,
-    "status: insight",
-    `created: ${today}`,
-    `updated: ${today}`,
-    category ? `category: ${category}` : "",
-    "---",
-    "",
-    `# ${title}`,
-    "",
-    body,
-    "",
-    category ? `*Category: ${category}*` : "",
-    "",
-    "---",
-    `*Captured: ${today}*`,
-    "",
-    "## Related",
-    "",
-    "_Add links to related pages._",
-    "",
-  ]
-    .filter((l) => l !== "")
-    .join("\n");
-  writeFileSync(sourcePagePath, pageContent, "utf-8");
+  const sourcePagePath = insightPath(paths, slug);
+
+  const pageBody = `# ${title}
+
+${body}
+
+${category ? `*Category: ${category}*` : ""}
+
+---
+*Captured: ${today}*
+
+## Related
+
+_Add links to related pages._`;
+
+  const doc = createKnowledgeDocument(
+    `sources/${slug}.md`,
+    {
+      type: "source",
+      title,
+      slug,
+      status: "insight",
+      created: today,
+      updated: today,
+      ...(category ? { category } : {}),
+    },
+    pageBody,
+  );
+  writeKnowledgeDocumentFile(sourcePagePath, doc);
 
   // Log event
   appendEvent(paths, {
@@ -125,22 +133,38 @@ export function registerWikiRetro(pi: ExtensionAPI, runtime?: Runtime): void {
     async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
       const paths = resolveVaultPaths(ctx.cwd ?? process.cwd());
 
-      if (!existsSync(join(paths.dotWiki, "config.json"))) {
+      const vaultCheck = inspectWritableVault(paths);
+      if (!vaultCheck.ok) {
         return {
           content: [
             {
               type: "text",
-              text: "No wiki vault found at this location. Initialize one with wiki_bootstrap first.",
+              text: `Wiki vault error: ${vaultCheck.diagnostics[0].message}`,
             },
           ],
-          details: { error: "no_vault" } as Record<string, unknown>,
+          details: {
+            error: vaultCheck.diagnostics[0].code,
+            diagnostics: vaultCheck.diagnostics,
+          } as Record<string, unknown>,
           isError: true,
         };
       }
 
-      const result = saveInsight(paths, params.slug, params.title, params.body, params.category, {
-        rebuild: !runtime,
-      });
+      let result: RetroResult;
+      try {
+        result = saveInsight(paths, params.slug, params.title, params.body, params.category, {
+          rebuild: !runtime,
+        });
+      } catch (error: unknown) {
+        if ((error as Error).message.startsWith("Invalid insight slug:")) {
+          return {
+            content: [{ type: "text", text: (error as Error).message }],
+            details: { error: "invalid_insight_slug" } as Record<string, unknown>,
+            isError: true,
+          };
+        }
+        throw error;
+      }
       if (runtime) {
         scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
       }

package/extensions/llm-wiki/lib/source-extractors.ts

@@ -1,7 +1,7 @@
 import { open } from "node:fs/promises";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-import { exec } from "./utils.js";
+import { type ExecApi, exec } from "./utils.js";
 
 export type ExtractionStatus = "success" | "failed" | "unsupported";
 
@@ -23,7 +23,7 @@ export interface FileExtractor {
 }
 
 interface FileExtractArgs {
-  pi: ExtensionAPI;
+  pi: ExecApi;
   filePath: string;
   content: string;
   signal?: AbortSignal;
@@ -35,7 +35,7 @@ interface UrlExtractor {
 }
 
 interface UrlExtractArgs {
-  pi: ExtensionAPI;
+  pi: ExecApi;
   url: string;
   signal?: AbortSignal;
 }
@@ -161,7 +161,7 @@ export function fileExtractorFor(filePath: string): FileExtractor {
 }
 
 export function extractUrlContent(
-  pi: ExtensionAPI,
+  pi: ExecApi,
   url: string,
   signal?: AbortSignal,
 ): Promise<ExtractedContent> {
@@ -197,7 +197,7 @@ function hasAnyExtension(extensions: string[]): (path: string) => boolean {
   return (path) => extensions.some((extension) => hasExtension(extension)(path));
 }
 
-async function extractPdf(pi: ExtensionAPI, source: string, signal?: AbortSignal): Promise<string> {
+async function extractPdf(pi: ExecApi, source: string, signal?: AbortSignal): Promise<string> {
   const extracted = await extractWithMarkItDown(pi, source, signal);
   return extracted || pdfExtractionFailureMessage(source);
 }
@@ -206,17 +206,13 @@ export function docxExtractionFailureMessage(source: string): string {
   return `_DOCX content could not be converted to markdown from ${source}. Ensure uvx and markitdown are installed._\n`;
 }
 
-async function extractDocx(
-  pi: ExtensionAPI,
-  source: string,
-  signal?: AbortSignal,
-): Promise<string> {
+async function extractDocx(pi: ExecApi, source: string, signal?: AbortSignal): Promise<string> {
   const extracted = await extractWithMarkItDown(pi, source, signal);
   return extracted || docxExtractionFailureMessage(source);
 }
 
 async function extractPdfUrl(
-  pi: ExtensionAPI,
+  pi: ExecApi,
   url: string,
   signal?: AbortSignal,
 ): Promise<ExtractedContent> {
@@ -232,7 +228,7 @@ async function extractPdfUrl(
 }
 
 async function extractTextUrl(
-  pi: ExtensionAPI,
+  pi: ExecApi,
   url: string,
   signal?: AbortSignal,
 ): Promise<ExtractedContent> {
@@ -266,13 +262,12 @@ async function extractTextUrl(
 }
 
 async function extractWithMarkItDown(
-  pi: ExtensionAPI,
+  pi: ExecApi,
   source: string,
   signal?: AbortSignal,
 ): Promise<string> {
-  if (!(await hasMarkItDown(pi, signal))) return "";
-
   try {
+    if (!(await hasMarkItDown(pi, signal))) return "";
     const mdResult = await exec(
       pi,
       "sh",
@@ -285,7 +280,7 @@ async function extractWithMarkItDown(
   }
 }
 
-async function hasMarkItDown(pi: ExtensionAPI, signal?: AbortSignal): Promise<boolean> {
+async function hasMarkItDown(pi: ExecApi, signal?: AbortSignal): Promise<boolean> {
   const markitdown = await exec(
     pi,
     "sh",
@@ -295,7 +290,7 @@ async function hasMarkItDown(pi: ExtensionAPI, signal?: AbortSignal): Promise<bo
   return markitdown.stdout.trim() === "yes";
 }
 
-async function fetchTextUrl(pi: ExtensionAPI, url: string, signal?: AbortSignal): Promise<string> {
+async function fetchTextUrl(pi: ExecApi, url: string, signal?: AbortSignal): Promise<string> {
   try {
     const curlResult = await exec(
       pi,