@zosmaai/pi-llm-wiki 0.8.2 → 0.9.0

package/extensions/llm-wiki/lib/tools.ts

@@ -2,6 +2,9 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { Type } from "typebox";
+import { launchEmbedPages, reindexEmbeddings, resolveEmbedder } from "./embeddings.js";
+import { scheduleReindex } from "./indexing.js";
+import { runIngestSynthesis } from "./ingest-worker.js";
 import {
   type Registry,
   appendEvent,
@@ -10,7 +13,9 @@ import {
   rebuildMetadata,
   rebuildMetadataLight,
 } from "./metadata.js";
+import type { Runtime } from "./runtime.js";
 import { captureFile, captureText, captureUrl } from "./source-packet.js";
+import { parseModelRef } from "./task-config.js";
 import {
   type VaultPaths,
   detectVaultFormat,
@@ -142,7 +147,7 @@ export function registerWikiBootstrap(pi: ExtensionAPI): void {
 
 // ─── 2. wiki_capture_source ─────────────────────────────
 
-export function registerWikiCaptureSource(pi: ExtensionAPI): void {
+export function registerWikiCaptureSource(pi: ExtensionAPI, runtime?: Runtime): void {
   pi.registerTool({
     name: "wiki_capture_source",
     label: "Wiki Capture Source",
@@ -191,7 +196,11 @@ export function registerWikiCaptureSource(pi: ExtensionAPI): void {
         };
       }
 
-      rebuildMetadataLight(paths);
+      if (runtime) {
+        scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
+      } else {
+        rebuildMetadataLight(paths);
+      }
 
       return {
         content: [
@@ -220,16 +229,17 @@ export function registerWikiCaptureSource(pi: ExtensionAPI): void {
 
 // ─── 3. wiki_ingest ─────────────────────────────────────
 
-export function registerWikiIngest(pi: ExtensionAPI): void {
+export function registerWikiIngest(pi: ExtensionAPI, runtime?: Runtime): void {
   pi.registerTool({
     name: "wiki_ingest",
     label: "Wiki Ingest",
     description:
-      "Process uningested source packets. Returns a batch of source IDs with extracted content for the LLM to synthesize.",
-    promptSnippet: "Ingest source packets: get batch of sources needing synthesis",
+      "Process uningested source packets. By default synthesis runs in the background (non-blocking) on the configured task model; pass background=false to return extracted content for the main agent to synthesize itself.",
+    promptSnippet: "Ingest source packets (background synthesis by default)",
     promptGuidelines: [
       "Use wiki_ingest when the user wants to process captured sources.",
-      "After calling this tool, read each source's extracted.md, update its source page, create entity/concept pages, and cross-reference.",
+      "By default ingestion runs in the BACKGROUND — you'll get a notification, not extracted content. Do NOT synthesize those sources yourself.",
+      "If the tool returns extracted content (background unavailable, or background=false), then read each source's extracted.md, update its source page, create entity/concept pages, and cross-reference.",
       "The extension auto-updates metadata — you do NOT need to edit meta/ files.",
     ],
     parameters: Type.Object({
@@ -237,7 +247,20 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
         Type.String({ description: "Specific source ID to ingest. Leave empty for all new." }),
       ),
       batch_size: Type.Optional(
-        Type.Number({ description: "Max sources to return (default: 3, max: 5)", default: 3 }),
+        Type.Number({ description: "Max sources to process (default: 3, max: 5)", default: 3 }),
+      ),
+      background: Type.Optional(
+        Type.Boolean({
+          description:
+            "Synthesize in the background without blocking (default: true). Set false to return extracted content for the main agent to synthesize.",
+          default: true,
+        }),
+      ),
+      model: Type.Optional(
+        Type.String({
+          description:
+            "Per-call model override as 'provider/id' (e.g. anthropic/claude-haiku). Overrides the configured wiki taskModel for this call; defaults to the configured/session model.",
+        }),
       ),
     }),
     async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
@@ -321,6 +344,75 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
         return { id, extracted, manifest };
       });
 
+      // ── Background synthesis (issue #65) ──────────────────
+      // Default path: dispatch each source to a background sub-agent so the
+      // main agent is not blocked. Falls back to the synchronous return below
+      // when no runtime/model is available (resolveModel ok:false).
+      const wantBackground = params.background !== false;
+      if (wantBackground && runtime) {
+        runtime.ensureConfig(ctx.cwd);
+        // Per-call model override (issue #69): 'provider/id' beats the
+        // configured taskModel; a malformed/unknown ref degrades to the
+        // configured/session model inside resolveModel.
+        const override = params.model ? parseModelRef(params.model) : undefined;
+        const resolved = await runtime.resolveModel(ctx, override);
+        if (resolved.ok) {
+          const launchCtx = { hasUI: ctx.hasUI, ui: ctx.ui };
+          for (const s of sources) {
+            runtime.launchTask(launchCtx, `ingest:${s.id}`, async () => {
+              const committed = await runIngestSynthesis({
+                model: resolved.model as Parameters<typeof runIngestSynthesis>[0]["model"],
+                apiKey: resolved.apiKey,
+                headers: resolved.headers,
+                paths,
+                sourceId: s.id,
+                manifest: s.manifest,
+                extracted: s.extracted,
+              });
+              if (committed) {
+                // Background semantic embeddings (#66): embed the pages this
+                // ingest just wrote, off-thread. No-op when unconfigured.
+                const pageIds = [
+                  `sources/${committed.sourceId}`,
+                  ...committed.entitiesCreated.map((e) => `entities/${e}`),
+                  ...committed.entitiesLinked.map((e) => `entities/${e}`),
+                  ...committed.conceptsCreated.map((c) => `concepts/${c}`),
+                  ...committed.conceptsLinked.map((c) => `concepts/${c}`),
+                ];
+                launchEmbedPages(runtime, launchCtx, paths, pageIds, `embed:ingest:${s.id}`);
+              }
+              if (ctx.hasUI) {
+                ctx.ui.notify(
+                  committed
+                    ? `LLM Wiki: ingested ${s.id} → ${committed.entitiesCreated.length} entit${committed.entitiesCreated.length === 1 ? "y" : "ies"}, ${committed.conceptsCreated.length} concept${committed.conceptsCreated.length === 1 ? "" : "s"}`
+                    : `LLM Wiki: ${s.id} produced no synthesis`,
+                  committed ? "info" : "warning",
+                );
+              }
+            });
+          }
+          return {
+            content: [
+              {
+                type: "text",
+                text: [
+                  `🔄 **Ingesting ${sources.length} source(s) in the background** (${toProcess.length - batch.length} remaining).`,
+                  "",
+                  ...sources.map((s) => `- **${s.id}**: ${s.manifest.title || s.id}`),
+                  "",
+                  "Synthesis runs on the configured task model without blocking. You'll be notified as each source completes — do NOT synthesize these yourself.",
+                ].join("\n"),
+              },
+            ],
+            details: {
+              background: true,
+              dispatched: sources.map((s) => s.id),
+              remaining: toProcess.length - batch.length,
+            } as Record<string, unknown>,
+          };
+        }
+      }
+
       return {
         content: [
           {
@@ -359,7 +451,7 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
 
 // ─── 4. wiki_ensure_page ────────────────────────────────
 
-export function registerWikiEnsurePage(pi: ExtensionAPI): void {
+export function registerWikiEnsurePage(pi: ExtensionAPI, runtime?: Runtime): void {
   pi.registerTool({
     name: "wiki_ensure_page",
     label: "Wiki Ensure Page",
@@ -426,6 +518,15 @@ export function registerWikiEnsurePage(pi: ExtensionAPI): void {
         path: `${folder}/${slug}`,
       });
 
+      // Register the new page so retrieval + embeddings can see it. When a
+      // background runtime is available, the rebuild + embeddings run off the
+      // tool's critical path; otherwise fall back to a synchronous rebuild.
+      if (runtime) {
+        scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
+      } else {
+        rebuildMetadataLight(paths);
+      }
+
       return {
         content: [{ type: "text", text: `✅ Created ${type} page: \`${pagePath}\`` }],
         details: { path: pagePath, created: true } as Record<string, unknown>,
@@ -854,6 +955,75 @@ export function registerWikiRebuildMeta(pi: ExtensionAPI): void {
 
 // ─── 9. wiki_log_event ──────────────────────────────────
 
+export function registerWikiReindexEmbeddings(pi: ExtensionAPI, runtime?: Runtime): void {
+  pi.registerTool({
+    name: "wiki_reindex_embeddings",
+    label: "Wiki Reindex Embeddings",
+    description:
+      "Backfill / refresh semantic embeddings for the vault. Embeds pages that " +
+      "are new or stale (content changed); pass force to re-embed everything. " +
+      "No-op when no embedding provider is configured.",
+    promptSnippet: "Backfill semantic embeddings for the wiki",
+    promptGuidelines: [
+      "Use wiki_reindex_embeddings to embed an existing vault or refresh stale embeddings.",
+      "Embeddings are optional: this no-ops cleanly when no embedding provider is configured.",
+    ],
+    parameters: Type.Object({
+      force: Type.Optional(
+        Type.Boolean({ description: "Re-embed every page, ignoring staleness (default: false)" }),
+      ),
+    }),
+    async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+      const paths = getPaths(ctx.cwd);
+      const vaultCheck = requireVault(paths);
+      if (!vaultCheck.ok) {
+        return {
+          content: [{ type: "text", text: vaultCheck.reason }],
+          details: { error: vaultCheck.reason } as Record<string, unknown>,
+          isError: true,
+        };
+      }
+
+      if (runtime) runtime.ensureConfig(ctx.cwd ?? paths.root);
+      const embedder = runtime ? resolveEmbedder(runtime.config) : undefined;
+      if (!embedder) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: 'ℹ️ No embedding provider configured — semantic embeddings are disabled. Set `llm-wiki.embeddingProvider` (e.g. "openai") in settings to enable.',
+            },
+          ],
+          details: { enabled: false } as Record<string, unknown>,
+        };
+      }
+
+      const stats = await reindexEmbeddings(paths, embedder, { force: params.force === true });
+      appendEvent(paths, {
+        kind: "reindex_embeddings",
+        embedded: stats.embedded,
+        skipped: stats.skipped,
+        pruned: stats.pruned,
+        model: embedder.model,
+      });
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: `✅ Embeddings reindexed (${embedder.model}): ${stats.embedded} embedded, ${stats.skipped} fresh, ${stats.pruned} pruned.`,
+          },
+        ],
+        details: {
+          enabled: true,
+          ...stats,
+          model: embedder.model,
+        } as Record<string, unknown>,
+      };
+    },
+  });
+}
+
 export function registerWikiLogEvent(pi: ExtensionAPI): void {
   pi.registerTool({
     name: "wiki_log_event",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
   "keywords": [
     "pi",

package/prompts/wiki-ingest.md

@@ -15,17 +15,20 @@ $ARGUMENTS
 
 ## Steps
 
-1. Call `wiki_ingest(source_id=<id if provided>, batch_size=3)` to get sources needing synthesis.
+1. Call `wiki_ingest(source_id=<id if provided>, batch_size=3)`.
 2. If the tool reports "All sources ingested", inform the user and stop.
-3. For each source in the returned batch:
+3. **If the tool reports it is ingesting in the background**, the synthesis sub-agent is handling those sources on the configured task model. Do NOT synthesize them yourself — just report which sources were dispatched and stop. (You'll be notified as each completes.)
+4. **Otherwise** (the tool returned extracted content — background unavailable or `background=false`), for each source in the returned batch:
    a. Read the extracted text from `raw/sources/<SOURCE_ID>/extracted.md`
    b. Update the skeleton source page in `wiki/sources/` with a proper summary, key entities, and concepts
    c. Use `wiki_ensure_page(type=entity, title=<name>)` for each new entity (people, orgs, tools, products)
    d. Use `wiki_ensure_page(type=concept, title=<name>)` for each new concept (ideas, patterns, frameworks)
    e. Add `[[wikilinks]]` cross-references between related pages
    f. Flag any contradictions with existing wiki content using `⚠️ **Contradiction**` markers
-4. After processing the batch, call `wiki_rebuild_meta` to update metadata.
-5. Report: "Ingested [N] sources → [M] pages created/updated. [X] contradictions flagged."
+5. After processing a synchronous batch, call `wiki_rebuild_meta` to update metadata.
+6. Report: "Ingested [N] sources → [M] pages created/updated. [X] contradictions flagged."
+
+> **Background vs synchronous:** ingestion runs in the background by default (non-blocking) when a task model is available, so the main agent is never stalled. It falls back to the synchronous main-agent flow above when no model/API key is configured, or when called with `background=false`.
 
 **Rules:**
 - Never modify files in `raw/` — source packets are immutable after capture.

package/skills/llm-wiki/SKILL.md

@@ -73,6 +73,20 @@ This searches both your **personal wiki** (`~/.llm-wiki/`) and the **project wik
 
 The extension also briefly searches automatically, but explicit calls with task-specific terms get better results.
 
+#### Two-Stage Recall (links-first for large vaults)
+
+Recall scales with vault size via **two-stage retrieval** (memex-style):
+
+- **Small vaults** (page count ≤ threshold): recall returns inline **content previews** — read them directly, no extra step.
+- **Large vaults** (page count > threshold): recall returns a **ranked list of links** only — `id`, `title`, `type`, `score`, and a 1-line snippet. **No full previews are injected**, to protect your context window.
+
+**The two-step contract for large vaults:**
+
+1. **Stage 1 — scan the links.** `wiki_recall` (and the auto-injected "Relevant Wiki Knowledge (links-first)" section) gives you ranked `[[id]]` links with scores and short snippets. Use the scores and snippets to pick the few pages that actually matter.
+2. **Stage 2 — expand on demand.** Call `read` (or `wiki_read`) on the chosen link **paths** to pull their full content. Do **not** assume the snippet is the whole page — open the link before relying on its content.
+
+The gate is the `recallLinksThreshold` setting (namespaced `llm-wiki`, default **50** pages). Page count is read from `meta/registry.json` (O(1), no page-body I/O). Set it to `0` to force links-first always, or a large number to always keep previews inline.
+
 ### At End — Save Insights with wiki_retro
 
 After completing any meaningful task, call `wiki_retro` to save key insights:
@@ -95,6 +109,22 @@ For thorough research, also use `wiki_search` to browse the full registry:
 wiki_search(query="broad topic")
 ```
 
+### Background Model Selection (`/wiki-model`)
+
+The wiki's background work (ingest synthesis, etc.) runs on a model you can choose. By **default it uses the current session model** — zero config.
+
+- **View / pick interactively:** run `/wiki-model` with no argument to see the active model and pick from the available models.
+- **Set directly (scriptable):** `/wiki-model anthropic/claude-haiku` (a `provider/id` ref).
+- **Revert to session model:** `/wiki-model session` (also `clear`/`default`/`reset`).
+
+The choice is persisted to project settings (`.pi/settings.json` under `llm-wiki.taskModel`) and shown in the status bar.
+
+**Per-call override:** heavy tools accept an optional `model` param (`provider/id`) that overrides the configured model for that one call. Precedence is **override > configured `taskModel` > session model**; an unknown ref degrades gracefully to the configured/session model. Example:
+
+```
+wiki_ingest(model="anthropic/claude-haiku")
+```
+
 ### Auto-Bootstrap (One-Time)
 
 The extension creates the wiki vault automatically on startup. On the first turn, it injects a directive asking you to infer topic and mode, then call: