@zosmaai/pi-llm-wiki 0.7.3 → 0.7.4

package/extensions/llm-wiki/lib/observation.ts

@@ -0,0 +1,310 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
+import { Type } from "typebox";
+import { appendEvent, rebuildMetadataLight } from "./metadata.js";
+import { type VaultPaths, fmtDate, resolveVaultPaths } from "./utils.js";
+
+// ─── Types ─────────────────────────────────────────────
+
+export interface ObservationInput {
+  /** Short descriptive title (≤80 chars) */
+  title: string;
+  /** The observation content — what happened, was decided, or was learned */
+  content: string;
+  /** Relevance level for retention priority */
+  relevance: "low" | "medium" | "high" | "critical";
+  /** Optional space-separated tags for categorization */
+  tags?: string;
+  /** Context: what was being worked on when this was observed */
+  source_context?: string;
+}
+
+export interface ObservationResult {
+  slug: string;
+  pagePath: string;
+}
+
+// ─── Save Observation ──────────────────────────────────
+
+const RELEVANCE_EMOJIS: Record<string, string> = {
+  low: "📝",
+  medium: "🔍",
+  high: "⭐",
+  critical: "🔴",
+};
+
+/**
+ * Save an observation as a wiki source page.
+ *
+ * Unlike wiki_retro (which saves atomic insights at task end),
+ * wiki_observe records timestamped observations during a session
+ * that can later be distilled into durable wiki pages.
+ *
+ * Observations are stored in wiki/sources/ with type: source and
+ * status: observation. They are searchable via wiki_recail.
+ */
+export function saveObservation(paths: VaultPaths, input: ObservationInput): ObservationResult {
+  const today = fmtDate();
+  const timestamp = new Date().toISOString();
+
+  // Generate a slug from title
+  const slugBase = input.title
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 60);
+  const slug = `obs-${today}-${slugBase}`;
+
+  // Write to wiki/sources/{slug}.md
+  const sourcePageDir = join(paths.wiki, "sources");
+  mkdirSync(sourcePageDir, { recursive: true });
+  const pagePath = join(sourcePageDir, `${slug}.md`);
+
+  const relevanceEmoji = RELEVANCE_EMOJIS[input.relevance] ?? "📝";
+  const tags = input.tags ?? "";
+  const sourceContext = input.source_context ?? "";
+
+  const pageContent = [
+    "---",
+    "type: source",
+    `title: "Observation: ${input.title}"`,
+    `slug: ${slug}`,
+    "status: observation",
+    `created: ${today}`,
+    `updated: ${today}`,
+    `relevance: ${input.relevance}`,
+    `observed_at: ${timestamp}`,
+    tags
+      ? `tags: [${tags
+          .split(/\s+/)
+          .filter(Boolean)
+          .map((t) => `"${t}"`)
+          .join(", ")}]`
+      : "",
+    sourceContext ? `source_context: "${sourceContext}"` : "",
+    "---",
+    "",
+    `# ${relevanceEmoji} Observation: ${input.title}`,
+    "",
+    input.content,
+    "",
+    `*Relevance: ${input.relevance}*`,
+    sourceContext ? `\n*Context: ${sourceContext}*` : "",
+    tags ? `\n*Tags: ${tags}*` : "",
+    "",
+    "---",
+    `*Observed: ${timestamp}*`,
+    "",
+  ]
+    .filter((l) => l !== "")
+    .join("\n");
+  writeFileSync(pagePath, pageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, {
+    kind: "observe",
+    slug,
+    title: input.title,
+    relevance: input.relevance,
+  });
+
+  // Rebuild metadata so the observation is immediately searchable
+  rebuildMetadataLight(paths);
+
+  return { slug, pagePath };
+}
+
+// ─── Shared Reminder State ────────────────────────────
+
+/**
+ * Mutable state shared between wiki_observe tool and the turn-end reminder.
+ * When the model calls wiki_observe, the tool sets observeDoneThisSession
+ * so the reminder stops nagging.
+ */
+export interface ReminderState {
+  observeDoneThisSession: boolean;
+}
+
+export function createReminderState(): ReminderState {
+  return { observeDoneThisSession: false };
+}
+
+// ─── Tool Registration ─────────────────────────────────
+
+/**
+ * Register the `wiki_observe` tool.
+ * The model calls this to record observations during a session.
+ * Observations are saved to the wiki and become searchable.
+ */
+export function registerWikiObserve(pi: ExtensionAPI, reminderState?: ReminderState): void {
+  pi.registerTool({
+    name: "wiki_observe",
+    label: "Wiki Observe",
+    description:
+      "Record an atomic observation from the current session into the wiki. " +
+      "Observations are timestamped, relevance-rated facts about decisions made, " +
+      "findings discovered, constraints established, or work completed. " +
+      "Saved observations are searchable via wiki_recall and can later be " +
+      "distilled into durable wiki pages via wiki_ensure_page. " +
+      "Call this proactively after non-trivial work — every observation " +
+      "compounds the wiki's knowledge across sessions.",
+    promptSnippet: "Record an observation about the current work",
+    promptGuidelines: [
+      "Call wiki_observe after non-trivial decisions, discoveries, or completions.",
+      "One observation per call. Use multiple calls for multiple observations.",
+      "Rate relevance honestly — most observations are medium or low, not critical.",
+      "Observations compound across sessions via wiki_recail.",
+    ],
+    parameters: Type.Object({
+      title: Type.String({
+        description:
+          "Short descriptive title (≤80 chars). Noun phrase, not a sentence. " +
+          "Example: 'JWT auth middleware added' or 'Postgres migration constraint discovered'",
+      }),
+      content: Type.String({
+        description:
+          "The observation in plain prose. What happened, was decided, or was learned. " +
+          "Preserve specific details: file paths, function names, error messages, " +
+          "quantitative results. Example: 'User decided to use JWT with refresh tokens. " +
+          "Implementation at src/auth/jwt.ts. Tests passing.'",
+      }),
+      relevance: Type.Union(
+        [
+          Type.Literal("low"),
+          Type.Literal("medium"),
+          Type.Literal("high"),
+          Type.Literal("critical"),
+        ],
+        {
+          description:
+            "Relevance level: low (routine), medium (task context), " +
+            "high (non-trivial decisions/constraints), critical (user identity, " +
+            "persistent preferences, completed work that must not be redone). " +
+            "Default: medium. Be honest — most observations are medium or low.",
+        },
+      ),
+      tags: Type.Optional(
+        Type.String({
+          description:
+            "Optional space-separated tags for categorization. " +
+            "Example: 'auth backend migration'",
+        }),
+      ),
+      source_context: Type.Optional(
+        Type.String({
+          description:
+            "What was being worked on. Example: 'Adding authentication module' or 'Debugging login timeout'",
+        }),
+      ),
+    }),
+    async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+      const paths = resolveVaultPaths(ctx.cwd ?? process.cwd());
+
+      if (!existsSync(join(paths.dotWiki, "config.json"))) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: "No wiki vault found at this location. Initialize one with wiki_bootstrap first.",
+            },
+          ],
+          details: { error: "no_vault" } as Record<string, unknown>,
+          isError: true,
+        };
+      }
+
+      const result = saveObservation(paths, {
+        title: params.title,
+        content: params.content,
+        relevance: params.relevance,
+        tags: params.tags,
+        source_context: params.source_context,
+      });
+
+      // Signal the reminder to stop nagging this session
+      if (reminderState) {
+        reminderState.observeDoneThisSession = true;
+      }
+
+      const relevanceEmoji = RELEVANCE_EMOJIS[params.relevance] ?? "📝";
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: [
+              `${relevanceEmoji} **Observation saved**: ${params.title}`,
+              "",
+              `- Page: \`${result.pagePath}\``,
+              `- Relevance: ${params.relevance}`,
+              params.tags ? `- Tags: ${params.tags}` : "",
+              "",
+              "This observation is now searchable via wiki_recall. " +
+                "It will compound with future observations across sessions.",
+            ]
+              .filter((l) => l !== "")
+              .join("\n"),
+          },
+        ],
+        details: {
+          slug: result.slug,
+          title: params.title,
+          relevance: params.relevance,
+          tags: params.tags || null,
+        } as Record<string, unknown>,
+      };
+    },
+  });
+}
+
+// ─── Turn-End Reminder ─────────────────────────────────
+
+/**
+ * Track observation cadence and send turn-end reminders.
+ * After every N significant turns, reminds the model to call wiki_observe
+ * for non-trivial findings (same pattern as memex-retro reminders).
+ */
+export function registerObservationReminder(
+  pi: ExtensionAPI,
+  reminderState: ReminderState,
+  options?: { turnsBetweenReminders?: number },
+): void {
+  const REMINDER_INTERVAL = options?.turnsBetweenReminders ?? 5;
+  let turnsSinceLastReminder = 0;
+
+  pi.on("session_start", async () => {
+    turnsSinceLastReminder = 0;
+    reminderState.observeDoneThisSession = false;
+  });
+
+  // After compaction, reset the reminder state so reminders resume
+  pi.on("session_compact", async () => {
+    turnsSinceLastReminder = 0;
+    reminderState.observeDoneThisSession = false;
+  });
+
+  pi.on("agent_end", async (_event, _ctx) => {
+    turnsSinceLastReminder++;
+    if (turnsSinceLastReminder < REMINDER_INTERVAL) return;
+    if (reminderState.observeDoneThisSession) return;
+
+    pi.sendMessage(
+      {
+        customType: "wiki-observe-reminder",
+        content: [
+          "**Wiki observation reminder:** If the work in this session produced non-trivial ",
+          "decisions, findings, constraints, or completions worth preserving across sessions,",
+          "call `wiki_observe` to record them. Observations are searchable via `wiki_recall`",
+          "and compound your wiki's knowledge over time.",
+          "",
+          "One observation per call. Separate distinct findings into multiple calls.",
+        ].join(" "),
+        display: false,
+      },
+      {
+        deliverAs: "nextTurn",
+      },
+    );
+  });
+}

package/extensions/llm-wiki/lib/recall.ts

@@ -27,9 +27,17 @@ export interface RecallResult {
   path: string;
   /** Vault source label for dual-vault results */
   vaultLabel?: string;
+  /** Relevance score (higher = better match). Used for filtering auto-injected results. */
+  score: number;
 }
 
-type Scored = { id: string; entry: Registry["pages"][string]; score: number; pagePath: string };
+type Scored = {
+  id: string;
+  entry: Registry["pages"][string];
+  score: number;
+  pagePath: string;
+  bestChunkPreview: string;
+};
 
 /**
  * Normalize text for recall matching.
@@ -116,16 +124,197 @@ function scoreField(value: unknown, terms: string[], weight: number): number {
   return score;
 }
 
+// ─── Common English stopwords ─────────────────────────
+
+const STOPWORDS = new Set([
+  "the",
+  "this",
+  "that",
+  "with",
+  "from",
+  "have",
+  "been",
+  "were",
+  "they",
+  "their",
+  "them",
+  "will",
+  "would",
+  "could",
+  "should",
+  "about",
+  "there",
+  "which",
+  "what",
+  "when",
+  "where",
+  "than",
+  "then",
+  "also",
+  "just",
+  "more",
+  "some",
+  "such",
+  "only",
+  "other",
+  "into",
+  "over",
+  "very",
+  "after",
+  "before",
+  "because",
+  "between",
+  "through",
+  "during",
+  "without",
+  "within",
+  "along",
+  "these",
+  "those",
+  "page",
+  "section",
+  "note",
+  "info",
+  "type",
+  "used",
+  "using",
+]);
+
+// ─── Chunk-Level Indexing ────────────────────────────
+
+interface PageChunk {
+  /** The heading line (e.g. "## Configuration") or empty for the intro section */
+  heading: string;
+  /** Content of this chunk */
+  content: string;
+  /** Heading level (0 for intro, 1 for #, 2 for ##, etc.) */
+  level: number;
+}
+
+/**
+ * Split a page's body into chunks by headings.
+ * Each heading and its following content become one chunk.
+ * Content before the first heading becomes the intro chunk.
+ */
+function chunkPage(body: string): PageChunk[] {
+  if (!body.trim()) return [];
+
+  const chunks: PageChunk[] = [];
+  const lines = body.split("\n");
+
+  let currentHeading = "";
+  let currentLevel = 0;
+  let currentContent: string[] = [];
+
+  for (const line of lines) {
+    const headingMatch = line.trim().match(/^(#{1,6})\s+(.+)$/);
+    if (headingMatch) {
+      // Save previous chunk
+      if (currentContent.length > 0 || currentHeading) {
+        chunks.push({
+          heading: currentHeading,
+          content: currentContent.join("\n").trim(),
+          level: currentLevel,
+        });
+      }
+      currentHeading = headingMatch[2].trim();
+      currentLevel = headingMatch[1].length;
+      currentContent = [];
+    } else {
+      currentContent.push(line);
+    }
+  }
+
+  // Save last chunk
+  if (currentContent.length > 0 || currentHeading) {
+    chunks.push({
+      heading: currentHeading,
+      content: currentContent.join("\n").trim(),
+      level: currentLevel,
+    });
+  }
+
+  return chunks;
+}
+
 function pagePreview(content: string): string {
   const { body } = parseFrontmatter(content);
   return body.trim().slice(0, 200).replace(/\n/g, " ");
 }
 
+/**
+ * Get a preview of the best-matching chunk, or fall back to the page intro.
+ * Shows the heading (if any) and the first ~200 chars of content.
+ */
+function chunkPreview(heading: string, content: string): string {
+  const trimmed = content.slice(0, 180).replace(/\n/g, " ");
+  if (heading) {
+    return `#${heading} — ${trimmed}`;
+  }
+  return trimmed;
+}
+
+/**
+ * Extract distinctive terms from the top search results for query expansion.
+ * Pseudo-relevance feedback: terms from top-matching pages that aren't in
+ * the original query become expansion candidates.
+ */
+function extractExpansionTerms(
+  scored: Scored[],
+  originalQuery: string,
+  paths: VaultPaths,
+  maxTerms = 6,
+): string[] {
+  const topResults = scored.slice(0, Math.min(3, scored.length));
+  if (topResults.length === 0) return [];
+
+  const originalNorm = normalizeText(originalQuery);
+  const termFreq = new Map<string, number>();
+
+  for (const { pagePath, entry } of topResults) {
+    // Collect text from registry metadata + file content
+    const metaText = normalizeText(
+      [entry.title, entry.aliases, entry.tags, entry.summary, entry.description]
+        .filter(Boolean)
+        .join(" "),
+    );
+    for (const w of metaText.split(/\s+/)) {
+      if (w.length >= 4 && !originalNorm.includes(w) && !STOPWORDS.has(w)) {
+        termFreq.set(w, (termFreq.get(w) || 0) + 1);
+      }
+    }
+
+    // Also extract from file body
+    if (existsSync(pagePath)) {
+      const content = readFileSync(pagePath, "utf-8");
+      const { body } = parseFrontmatter(content);
+      const bodyNorm = normalizeText(body);
+      for (const w of bodyNorm.split(/\s+/)) {
+        if (w.length >= 4 && !originalNorm.includes(w) && !STOPWORDS.has(w)) {
+          termFreq.set(w, (termFreq.get(w) || 0) + 1);
+        }
+      }
+    }
+  }
+
+  // Sort by frequency descending, take top N
+  return Array.from(termFreq.entries())
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .slice(0, maxTerms)
+    .map(([term]) => term);
+}
+
 /**
  * Search a single vault's registry for pages matching a query.
  * Returns up to `maxResults` matches, each with a content preview.
+ * Results below `minScore` are excluded (default 0 = no filtering).
  */
-export function searchWiki(paths: VaultPaths, query: string, maxResults = 5): RecallResult[] {
+export function searchWiki(
+  paths: VaultPaths,
+  query: string,
+  maxResults = 5,
+  minScore = 0,
+): RecallResult[] {
   const registry = readJson<Registry>(join(paths.meta, "registry.json"), {
     version: "1.0",
     last_updated: "",
@@ -169,26 +358,83 @@ export function searchWiki(paths: VaultPaths, query: string, maxResults = 5): Re
     score += scoreField(frontmatter.category, terms, 2);
     score += scoreField(frontmatter.domain, terms, 2);
 
-    // Body search makes the wiki useful even when registry metadata is sparse.
-    // Headings get a stronger boost because they are human-authored labels.
-    const headings = body
-      .split("\n")
-      .filter((line) => line.trim().startsWith("#"))
-      .join(" ");
-    score += scoreField(headings, terms, 4);
-    score += scoreField(body, terms, 1);
+    // Body search: use chunk-level indexing for more precise matching.
+    // Each section of the page is scored independently, so a query about
+    // "Postgres" matches only the Postgres section, not the whole page.
+    let bestChunkScore = 0;
+    let bestChunkHeading = "";
+    let bestChunkContent = "";
+
+    if (body.trim()) {
+      const chunks = chunkPage(body);
+      for (const chunk of chunks) {
+        let chunkScore = 0;
+        // Heading gets a strong boost
+        chunkScore += scoreField(chunk.heading, terms, 4);
+        // Chunk body content
+        chunkScore += scoreField(chunk.content, terms, 1);
+
+        if (chunkScore > bestChunkScore) {
+          bestChunkScore = chunkScore;
+          bestChunkHeading = chunk.heading;
+          bestChunkContent = chunk.content;
+        }
+      }
+    }
+
+    // Add best chunk score to total page score
+    score += bestChunkScore;
 
     if (score > 0) {
-      scored.push({ id, entry, score, pagePath });
+      scored.push({
+        id,
+        entry,
+        score,
+        pagePath,
+        bestChunkPreview: bestChunkContent ? chunkPreview(bestChunkHeading, bestChunkContent) : "",
+      });
     }
   }
 
   scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
-  const top = scored.slice(0, maxResults);
 
-  return top.map(({ id, entry, pagePath }) => {
-    let preview = "";
-    if (existsSync(pagePath)) {
+  // ── Pseudo-Relevance Feedback (PRF) ─────────────────
+  // Extract distinctive terms from the top 3 results and use them to
+  // boost semantically related pages. This gives "semantic" expansion
+  // without external dependencies: if an "Authentication" page mentions
+  // JWT, OAuth, and sessions, those terms boost other pages that discuss
+  // related concepts.
+  const expansionTerms = extractExpansionTerms(scored, query, paths, 6);
+  if (expansionTerms.length > 0) {
+    const expTermList = queryTerms(expansionTerms.join(" "));
+    // Apply expansion scoring to the top 25 results (cheap re-read)
+    const expansionCandidates = scored.slice(0, Math.min(25, scored.length));
+    for (const item of expansionCandidates) {
+      const content = existsSync(item.pagePath) ? readFileSync(item.pagePath, "utf-8") : "";
+      const { body } = parseFrontmatter(content);
+      let expChunkScore = 0;
+      if (body.trim()) {
+        const chunks = chunkPage(body);
+        for (const chunk of chunks) {
+          let cs = 0;
+          cs += scoreField(chunk.heading, expTermList, 2); // half weight
+          cs += scoreField(chunk.content, expTermList, 0.5);
+          if (cs > expChunkScore) expChunkScore = cs;
+        }
+      }
+      // Dampened addition — expansion contributes at most 40%
+      item.score += expChunkScore * 0.4;
+    }
+  }
+
+  // Re-sort after expansion scoring
+  scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+  const top = scored.filter((s) => s.score >= minScore).slice(0, maxResults);
+
+  return top.map(({ id, entry, pagePath, score, bestChunkPreview }) => {
+    let preview = bestChunkPreview;
+    if (!preview && existsSync(pagePath)) {
+      // Fallback: no chunk matched, show page intro
       preview = pagePreview(readFileSync(pagePath, "utf-8"));
     }
 
@@ -198,6 +444,7 @@ export function searchWiki(paths: VaultPaths, query: string, maxResults = 5): Re
       type: String(entry.type || "page"),
       preview,
       path: pagePath,
+      score,
     };
   });
 }
@@ -205,23 +452,32 @@ export function searchWiki(paths: VaultPaths, query: string, maxResults = 5): Re
 /**
  * Search both project/primary vault and personal vault, merging results.
  * Personal results are appended after primary results, deduplicated by page ID.
+ *
+ * @param minScore - Minimum relevance score (default 0 = no filter).
+ * @param includePersonal - Whether to search the personal vault (default true).
+ *   Auto-injection should pass false to avoid personal-vault contamination.
  */
 export function searchWikiLayered(
   primaryPaths: VaultPaths,
   query: string,
   maxResults = 5,
+  minScore = 0,
+  includePersonal = true,
 ): RecallResult[] {
   // Search primary vault
-  const primaryResults = searchWiki(primaryPaths, query, maxResults);
+  const primaryResults = searchWiki(primaryPaths, query, maxResults, minScore);
 
   // If primary is already the personal vault, no layered search needed
   if (isPersonalVault(primaryPaths)) return primaryResults;
 
-  // Search personal vault as secondary layer
-  const personalPaths = getPersonalWikiPaths();
-  if (!existsSync(join(personalPaths.dotWiki, "config.json"))) return primaryResults;
-
-  const personalResults = searchWiki(personalPaths, query, maxResults);
+  // Search personal vault as secondary layer (only when explicitly requested)
+  let personalResults: RecallResult[] = [];
+  if (includePersonal) {
+    const personalPaths = getPersonalWikiPaths();
+    if (existsSync(join(personalPaths.dotWiki, "config.json"))) {
+      personalResults = searchWiki(personalPaths, query, maxResults, minScore);
+    }
+  }
 
   // Merge: personal results first (they're the user's accumulated knowledge),
   // then primary results (project-specific). Deduplicate by page ID.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
   "keywords": [
     "pi",