@zosmaai/pi-llm-wiki 0.10.7 → 0.11.0

package/extensions/llm-wiki/lib/vault-format.ts

@@ -0,0 +1,363 @@
+import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs";
+import { isAbsolute, join, relative, sep } from "node:path";
+import {
+  type KnowledgeDiagnostic,
+  type KnowledgeDocument,
+  parseKnowledgeDocument,
+  parseMarkdownFrontmatter,
+} from "./knowledge-document.js";
+import { type VaultPaths, relativePhysicalPath } from "./utils.js";
+
+export type KnowledgeFormat = "legacy" | "okf-0.2";
+
+export interface VaultFormatState {
+  knowledgeFormat: KnowledgeFormat;
+  diagnostics: KnowledgeDiagnostic[];
+  blocking: boolean;
+}
+
+export interface DiscoveredDocument extends KnowledgeDocument {
+  absolutePath: string;
+}
+
+export interface DiscoveryResult {
+  documents: DiscoveredDocument[];
+  diagnostics: KnowledgeDiagnostic[];
+  blocking: boolean;
+}
+
+export class VaultWriteError extends Error {
+  constructor(readonly diagnostics: KnowledgeDiagnostic[]) {
+    super(diagnostics[0]?.message ?? "Wiki vault is not writable");
+    this.name = "VaultWriteError";
+  }
+}
+
+const RESERVED_NAMES = new Set(["index", "log"]);
+
+export function compareCodePoint(a: string, b: string): number {
+  const left = a.normalize("NFC");
+  const right = b.normalize("NFC");
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+function diag(
+  severity: "warning" | "error",
+  code: KnowledgeDiagnostic["code"],
+  path: string,
+  message: string,
+): KnowledgeDiagnostic {
+  return { severity, code, path, message };
+}
+
+function isReservedName(filename: string): boolean {
+  const name = filename.toLowerCase();
+  return name === "index.md" || name === "log.md";
+}
+
+export type VaultConfigRead =
+  | { ok: true; config: Record<string, unknown> }
+  | { ok: false; diagnostic: KnowledgeDiagnostic };
+
+export function readVaultConfig(paths: VaultPaths): VaultConfigRead {
+  const path = join(paths.dotWiki, "config.json");
+  try {
+    const parsed: unknown = JSON.parse(readFileSync(path, "utf8"));
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new Error("config.json must contain an object");
+    }
+    return { ok: true, config: parsed as Record<string, unknown> };
+  } catch (error: unknown) {
+    return {
+      ok: false,
+      diagnostic: diag(
+        "error",
+        "config_invalid_knowledge_format",
+        "config.json",
+        `Cannot read valid wiki config: ${(error as Error).message}`,
+      ),
+    };
+  }
+}
+
+export function inspectVaultFormat(paths: VaultPaths): VaultFormatState {
+  const diagnostics: KnowledgeDiagnostic[] = [];
+  let blocking = false;
+
+  const configResult = readVaultConfig(paths);
+  if (!configResult.ok) {
+    return {
+      knowledgeFormat: "legacy",
+      diagnostics: [configResult.diagnostic],
+      blocking: true,
+    };
+  }
+
+  const config = configResult.config;
+  const rawFormat = config.knowledge_format;
+
+  // Resolve format
+  let format: KnowledgeFormat;
+  if (rawFormat === undefined) {
+    format = "legacy";
+  } else if (rawFormat === "legacy" || rawFormat === "okf-0.2") {
+    format = rawFormat;
+  } else {
+    return {
+      knowledgeFormat: "legacy",
+      diagnostics: [
+        diag(
+          "error",
+          "config_invalid_knowledge_format",
+          "config.json",
+          `Invalid knowledge_format value: ${JSON.stringify(rawFormat)}`,
+        ),
+      ],
+      blocking: true,
+    };
+  }
+
+  // In OKF mode, check root index version
+  // Missing root index is repairable; version mismatch blocks until explicitly handled
+  if (format === "okf-0.2") {
+    const rootIndexPath = join(paths.wiki, "index.md");
+    let rootContent: string | undefined;
+    try {
+      rootContent = readFileSync(rootIndexPath, "utf8");
+    } catch (error: unknown) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        diagnostics.push(
+          diag(
+            "error",
+            "okf_version_mismatch",
+            "wiki/index.md",
+            `Cannot read OKF root index: ${(error as Error).message}`,
+          ),
+        );
+        blocking = true;
+      }
+    }
+    if (rootContent !== undefined) {
+      const frontmatter = parseMarkdownFrontmatter(rootContent, "index.md");
+      if (!frontmatter.ok || frontmatter.mapping.okf_version !== "0.2") {
+        diagnostics.push(
+          diag(
+            "error",
+            "okf_version_mismatch",
+            "wiki/index.md",
+            frontmatter.ok
+              ? 'OKF root index must declare okf_version "0.2"'
+              : `Malformed OKF root index: ${frontmatter.diagnostics[0].message}`,
+          ),
+        );
+        blocking = true;
+      }
+    }
+  }
+
+  // In legacy mode, check if root index declares unsupported version
+  if (format === "legacy") {
+    const rootIndexPath = join(paths.wiki, "index.md");
+    try {
+      const content = readFileSync(rootIndexPath, "utf8");
+      const frontmatter = parseMarkdownFrontmatter(content, "index.md");
+      if (frontmatter.ok && frontmatter.mapping.okf_version !== undefined) {
+        if (frontmatter.mapping.okf_version !== "0.2") {
+          diagnostics.push(
+            diag(
+              "error",
+              "okf_version_mismatch",
+              "wiki/index.md",
+              `Root index declares unsupported okf_version "${frontmatter.mapping.okf_version}"`,
+            ),
+          );
+          blocking = true;
+        }
+      }
+    } catch {
+      // No root index in legacy mode - fine
+    }
+  }
+
+  return {
+    knowledgeFormat: format,
+    diagnostics,
+    blocking,
+  };
+}
+
+interface MarkdownScan {
+  files: string[];
+  diagnostics: KnowledgeDiagnostic[];
+}
+
+function collectMarkdownFiles(dir: string, wikiRoot: string): MarkdownScan {
+  const files: string[] = [];
+  const diagnostics: KnowledgeDiagnostic[] = [];
+  let entries: string[];
+  try {
+    entries = readdirSync(dir).sort(compareCodePoint);
+  } catch (error: unknown) {
+    diagnostics.push(
+      diag(
+        "error",
+        "frontmatter_parse_error",
+        relative(wikiRoot, dir).replace(/\\/g, "/") || ".",
+        `Failed to scan knowledge directory: ${(error as Error).message}`,
+      ),
+    );
+    return { files, diagnostics };
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry);
+    try {
+      const stat = lstatSync(fullPath);
+      if (stat.isSymbolicLink()) continue;
+      if (stat.isDirectory()) {
+        const child = collectMarkdownFiles(fullPath, wikiRoot);
+        files.push(...child.files);
+        diagnostics.push(...child.diagnostics);
+      } else if (stat.isFile() && entry.toLowerCase().endsWith(".md") && !isReservedName(entry)) {
+        files.push(fullPath);
+      }
+    } catch (error: unknown) {
+      diagnostics.push(
+        diag(
+          "error",
+          "frontmatter_parse_error",
+          relative(wikiRoot, fullPath).replace(/\\/g, "/"),
+          `Failed to inspect knowledge path: ${(error as Error).message}`,
+        ),
+      );
+    }
+  }
+  return { files, diagnostics };
+}
+
+/** Validate vault is writable: exists, valid mode, no blocking version mismatch. */
+export function inspectWritableVault(
+  paths: VaultPaths,
+): { ok: true; format: KnowledgeFormat } | { ok: false; diagnostics: KnowledgeDiagnostic[] } {
+  // Check vault exists
+  if (!existsSync(paths.dotWiki)) {
+    return {
+      ok: false,
+      diagnostics: [
+        diag(
+          "error",
+          "config_invalid_knowledge_format",
+          paths.dotWiki,
+          "Wiki vault not found. Run wiki_bootstrap first.",
+        ),
+      ],
+    };
+  }
+  const state = inspectVaultFormat(paths);
+  if (state.blocking) {
+    return { ok: false, diagnostics: state.diagnostics };
+  }
+  return { ok: true, format: state.knowledgeFormat };
+}
+
+/** Assert that an authoritative writer may mutate this vault. */
+export function assertWritableVault(paths: VaultPaths): KnowledgeFormat {
+  const result = inspectWritableVault(paths);
+  if (!result.ok) throw new VaultWriteError(result.diagnostics);
+  return result.format;
+}
+
+/** Check if path is a generated OKF reserved file (mode-aware). */
+export function isGeneratedOkfPath(path: string, paths: VaultPaths): boolean {
+  const state = inspectVaultFormat(paths);
+  if (state.blocking || state.knowledgeFormat !== "okf-0.2") return false;
+  const rel = relativePhysicalPath(paths.wiki, path);
+  if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return false;
+  const parts = rel.split(sep);
+  const name = parts.at(-1)?.toLowerCase();
+  return name === "index.md" || (parts.length === 1 && name === "log.md");
+}
+
+export function discoverKnowledgeDocuments(paths: VaultPaths): DiscoveryResult {
+  const diagnostics: KnowledgeDiagnostic[] = [];
+  const documents: DiscoveredDocument[] = [];
+  let blocking = false;
+  const seenIds = new Map<string, { id: string; physicalPath: string }>();
+
+  const scan = collectMarkdownFiles(paths.wiki, paths.wiki);
+  diagnostics.push(...scan.diagnostics);
+  if (scan.diagnostics.length > 0) blocking = true;
+
+  for (const file of scan.files) {
+    const physicalPath = relative(paths.wiki, file).replace(/\\/g, "/");
+    const normalizedPath = physicalPath.normalize("NFC");
+    const id = normalizedPath.replace(/\.md$/, "");
+
+    // Check for reserved names (case-insensitive)
+    const filename = (normalizedPath.split("/").pop() ?? "").replace(/\.md$/i, "").toLowerCase();
+    if (RESERVED_NAMES.has(filename)) {
+      continue;
+    }
+
+    // Check for identity collision
+    const collisionKey = id.toLowerCase();
+    const existing = seenIds.get(collisionKey);
+    if (existing && existing.physicalPath !== physicalPath) {
+      diagnostics.push(
+        diag(
+          "error",
+          "concept_identity_collision",
+          physicalPath,
+          `Identity collision between ${existing.physicalPath} and ${physicalPath}`,
+        ),
+      );
+      blocking = true;
+      continue;
+    }
+    seenIds.set(collisionKey, { id, physicalPath });
+
+    // Parse the document
+    try {
+      const content = readFileSync(file, "utf8");
+      const result = parseKnowledgeDocument(content, normalizedPath);
+
+      if (!result.ok) {
+        diagnostics.push(...result.diagnostics);
+        blocking = true;
+        continue;
+      }
+
+      const doc = result.document;
+      if (result.diagnostics.length > 0) {
+        diagnostics.push(...result.diagnostics);
+      }
+
+      documents.push({
+        ...doc,
+        id,
+        path: normalizedPath,
+        absolutePath: file,
+      });
+    } catch (e: unknown) {
+      const err = e as Error;
+      diagnostics.push(
+        diag(
+          "error",
+          "frontmatter_parse_error",
+          normalizedPath,
+          `Failed to read file: ${err.message}`,
+        ),
+      );
+      blocking = true;
+    }
+  }
+
+  // Sort documents by code point order
+  documents.sort((a, b) => compareCodePoint(a.id, b.id));
+
+  return {
+    documents,
+    diagnostics,
+    blocking,
+  };
+}

package/extensions/llm-wiki/lib/wiki-service.ts

@@ -0,0 +1,183 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import type { KnowledgeDiagnostic } from "./knowledge-document.js";
+import type { Registry } from "./metadata.js";
+import { readJson } from "./utils.js";
+import type { VaultPaths } from "./utils.js";
+import type { KnowledgeFormat } from "./vault-format.js";
+import {
+  compareCodePoint,
+  discoverKnowledgeDocuments,
+  inspectVaultFormat,
+} from "./vault-format.js";
+
+/**
+ * Shared wiki service functions consumed by Pi and MCP.
+ *
+ * These are pure adapters over the shared document model and registry.
+ * No YAML parsing, file scanning, or page serialization here.
+ */
+
+export interface RegistrySearchResult {
+  matches: Array<{ id: string; title: string; type: string }>;
+  diagnostics: KnowledgeDiagnostic[];
+}
+
+export interface WikiStatusSnapshot {
+  knowledgeFormat: KnowledgeFormat;
+  totalPages: number;
+  byType: Record<string, number>;
+  blockingDiagnostics: KnowledgeDiagnostic[];
+  lastUpdated: string;
+}
+
+/**
+ * Search the registry for matching concepts.
+ *
+ * Matches ID, semantic title, type, category, domain, tags, aliases, and recall triggers.
+ * Preserves unknown types as strings.
+ */
+export function searchRegistry(
+  paths: VaultPaths,
+  query: string,
+  typeFilter?: string,
+): RegistrySearchResult {
+  const diagnostics: KnowledgeDiagnostic[] = [];
+  const vaultState = inspectVaultFormat(paths);
+  diagnostics.push(...vaultState.diagnostics);
+
+  const registryPath = join(paths.meta, "registry.json");
+  if (!existsSync(registryPath)) {
+    return { matches: [], diagnostics };
+  }
+
+  const registry = readJson<Registry>(registryPath, {
+    version: "1.0",
+    last_updated: "",
+    pages: {},
+  });
+  const normalizedQuery = query.toLowerCase();
+
+  const matches: Array<{ id: string; title: string; type: string }> = [];
+
+  for (const [id, entry] of Object.entries(registry.pages)) {
+    // Apply type filter
+    if (typeFilter && String(entry.type).toLowerCase() !== typeFilter.toLowerCase()) {
+      continue;
+    }
+
+    // Check if query matches any searchable field
+    if (matchesField(id, entry, normalizedQuery)) {
+      matches.push({
+        id,
+        title: String(entry.title || id),
+        type: String(entry.type || "unknown"),
+      });
+    }
+  }
+
+  // Sort by code point order for determinism
+  matches.sort((a, b) => compareCodePoint(a.id, b.id));
+
+  return { matches, diagnostics };
+}
+
+function matchesField(id: string, entry: Record<string, unknown>, query: string): boolean {
+  // Match ID
+  if (id.toLowerCase().includes(query)) return true;
+
+  // Match title
+  if (
+    String(entry.title || "")
+      .toLowerCase()
+      .includes(query)
+  )
+    return true;
+
+  // Match type
+  if (
+    String(entry.type || "")
+      .toLowerCase()
+      .includes(query)
+  )
+    return true;
+
+  // Match category/domain
+  if (
+    String(entry.category || "")
+      .toLowerCase()
+      .includes(query)
+  )
+    return true;
+  if (
+    String(entry.domain || "")
+      .toLowerCase()
+      .includes(query)
+  )
+    return true;
+
+  // Match tags (array or string)
+  const tags = entry.tags;
+  if (Array.isArray(tags)) {
+    if (tags.some((t) => String(t).toLowerCase().includes(query))) return true;
+  } else if (typeof tags === "string" && tags.toLowerCase().includes(query)) {
+    return true;
+  }
+
+  // Match aliases (array or string)
+  const aliases = entry.aliases;
+  if (Array.isArray(aliases)) {
+    if (aliases.some((a) => String(a).toLowerCase().includes(query))) return true;
+  } else if (typeof aliases === "string" && aliases.toLowerCase().includes(query)) {
+    return true;
+  }
+
+  // Match recall_triggers (array or string)
+  const triggers = entry.recall_triggers;
+  if (Array.isArray(triggers)) {
+    if (triggers.some((t) => String(t).toLowerCase().includes(query))) return true;
+  } else if (typeof triggers === "string" && triggers.toLowerCase().includes(query)) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Get a status snapshot of the wiki.
+ *
+ * Reports resolved knowledge_format, page counts, and blocking diagnostics.
+ */
+export function getWikiStatus(paths: VaultPaths): WikiStatusSnapshot {
+  const vaultState = inspectVaultFormat(paths);
+  const diagnostics = [...vaultState.diagnostics];
+
+  // Also check discovery for current concept health
+  const discovery = discoverKnowledgeDocuments(paths);
+  diagnostics.push(...discovery.diagnostics);
+
+  const registryPath = join(paths.meta, "registry.json");
+  let registry: Registry | undefined;
+  if (existsSync(registryPath)) {
+    registry = readJson<Registry>(registryPath, { version: "1.0", last_updated: "", pages: {} });
+  }
+
+  const byType: Record<string, number> = {};
+  let totalPages = 0;
+
+  if (registry) {
+    for (const entry of Object.values(registry.pages)) {
+      totalPages++;
+      const type = String(entry.type || "unknown");
+      byType[type] = (byType[type] || 0) + 1;
+    }
+  }
+
+  return {
+    knowledgeFormat: vaultState.knowledgeFormat,
+    totalPages,
+    byType,
+    blockingDiagnostics: diagnostics.filter((d) => d.severity === "error"),
+    lastUpdated: registry?.last_updated || "",
+  };
+}

package/mcp/exec.ts

@@ -0,0 +1,122 @@
+import { execFile, spawn } from "node:child_process";
+import type { ExecApi } from "../extensions/llm-wiki/lib/utils.js";
+
+const MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
+
+export function createExecApi(): ExecApi {
+  return {
+    exec(command, args, options = {}) {
+      return new Promise((resolve) => {
+        let killed = false;
+        let settled = false;
+        let stdoutLimited = false;
+        let stderrLimited = false;
+        let forceTimer: NodeJS.Timeout | undefined;
+        let stdout = "";
+        let stderr = "";
+        const child = spawn(command, args, {
+          cwd: options.cwd,
+          stdio: ["ignore", "pipe", "pipe"],
+          detached: process.platform !== "win32",
+        });
+
+        const sendSignal = (signal: "SIGTERM" | "SIGKILL") => {
+          if (process.platform === "win32" && child.pid) {
+            execFile("taskkill", ["/pid", String(child.pid), "/T", "/F"], () => {});
+            return;
+          }
+          if (child.pid) {
+            try {
+              process.kill(-child.pid, signal);
+              return;
+            } catch {
+              // Fall back to the direct child when process-group signalling fails.
+            }
+          }
+          child.kill(signal);
+        };
+        const forceStop = () => {
+          if (forceTimer) clearTimeout(forceTimer);
+          forceTimer = undefined;
+          sendSignal("SIGKILL");
+        };
+        const stop = () => {
+          if (killed) return;
+          killed = true;
+          sendSignal("SIGTERM");
+          if (process.platform !== "win32") forceTimer = setTimeout(forceStop, 100);
+        };
+        const appendOutput = (
+          current: string,
+          chunk: string,
+          limited: boolean,
+        ): [string, boolean] => {
+          if (limited) return [current, true];
+          const remaining = MAX_OUTPUT_BYTES - Buffer.byteLength(current);
+          if (remaining <= 0) {
+            stop();
+            return [current, true];
+          }
+          const bytes = Buffer.from(chunk);
+          if (bytes.byteLength <= remaining) return [current + chunk, false];
+          stop();
+          let prefix = "";
+          let prefixBytes = 0;
+          for (const char of chunk) {
+            const charBytes = Buffer.byteLength(char);
+            if (prefixBytes + charBytes > remaining) break;
+            prefix += char;
+            prefixBytes += charBytes;
+          }
+          return [current + prefix, true];
+        };
+
+        child.stdout?.setEncoding("utf8");
+        child.stderr?.setEncoding("utf8");
+        child.stdout?.on("data", (chunk: string) => {
+          [stdout, stdoutLimited] = appendOutput(stdout, chunk, stdoutLimited);
+        });
+        child.stderr?.on("data", (chunk: string) => {
+          [stderr, stderrLimited] = appendOutput(stderr, chunk, stderrLimited);
+        });
+
+        const timer = options.timeout ? setTimeout(stop, options.timeout) : undefined;
+        const abort = () => stop();
+        options.signal?.addEventListener("abort", abort, { once: true });
+
+        const finish = (code: number) => {
+          if (settled) return;
+          if (forceTimer) {
+            clearTimeout(forceTimer);
+            forceTimer = undefined;
+            if (child.pid) {
+              try {
+                process.kill(-child.pid, "SIGKILL");
+              } catch {
+                // The process group already exited; do not signal the closed child's stale PID.
+              }
+            }
+          }
+          cleanup();
+          resolve({
+            stdout,
+            stderr,
+            code: stdoutLimited || stderrLimited ? 1 : killed && code === 0 ? 1 : code,
+            killed,
+          });
+        };
+        child.once("error", () => finish(1));
+        child.once("close", (code) => finish(typeof code === "number" ? code : 1));
+
+        function cleanup() {
+          settled = true;
+          if (timer) clearTimeout(timer);
+          if (forceTimer) clearTimeout(forceTimer);
+          options.signal?.removeEventListener("abort", abort);
+        }
+
+        if (options.signal?.aborted) stop();
+      });
+    },
+  };
+}