@zosmaai/pi-llm-wiki 0.8.2 → 0.9.0

package/extensions/llm-wiki/lib/indexing.ts

@@ -0,0 +1,88 @@
+/**
+ * Non-blocking vault (re)indexing.
+ *
+ * Writing a wiki page (observe / retro / capture / ensure_page) and editing one
+ * by hand both require the derived metadata — `meta/registry.json`,
+ * `backlinks.json`, `index.md`, `log.md` — to be rebuilt, plus (optionally) the
+ * semantic embedding store to be refreshed. That rebuild is O(pages): it
+ * rescans every page in the vault. Doing it inline on the tool's / turn's
+ * critical path makes every write get slower as the vault grows.
+ *
+ * `scheduleReindex` moves that work off the caller's stack onto the shared
+ * background Runtime (#64) and coalesces a burst of writes into a single pass:
+ *
+ *   - A leading micro-yield guarantees the caller (a tool's `execute`, or the
+ *     turn_end handler) returns BEFORE the heavy rebuild runs.
+ *   - A per-vault `dirty` flag + drain loop means writes that land while a pass
+ *     is in flight are folded into a trailing rebuild instead of being lost —
+ *     this also covers the async window of the embeddings refresh.
+ *   - A per-vault `inflight` guard collapses concurrent schedule calls onto the
+ *     same promise (single-flight), so N writes in a turn cost one rebuild.
+ *
+ * Errors are isolated by `Runtime.launchTask`; the promise never rejects. The
+ * embeddings step is a no-op unless an embedder is configured (#66/#67).
+ */
+
+import { reindexEmbeddings, resolveEmbedder } from "./embeddings.js";
+import { rebuildMetadataLight } from "./metadata.js";
+import type { LaunchCtx, Runtime } from "./runtime.js";
+import type { VaultPaths } from "./utils.js";
+
+/** Promise of the current background pass, keyed by vault root. */
+const inflight = new Map<string, Promise<void>>();
+/** Vault roots with writes awaiting a (re)build. */
+const dirty = new Set<string>();
+
+/** Stable single-flight label for a vault's background index pass. */
+export function indexLabel(root: string): string {
+  return `index:${root}`;
+}
+
+/**
+ * Schedule a non-blocking metadata rebuild (+ embeddings refresh) for a vault.
+ * Returns the promise of the in-flight pass so callers/tests can await drainage
+ * (the agent loop itself never awaits it). Safe to call on every write.
+ */
+export function scheduleReindex(
+  runtime: Runtime,
+  ctx: LaunchCtx,
+  paths: VaultPaths,
+): Promise<void> {
+  const root = paths.root;
+  dirty.add(root);
+
+  const active = inflight.get(root);
+  if (active) return active;
+
+  const pass = runtime.launchTask(ctx, indexLabel(root), async () => {
+    // Yield once so the caller returns before the O(pages) rebuild runs. This
+    // is what makes the surrounding write non-blocking.
+    await Promise.resolve();
+    try {
+      // Drain: keep rebuilding until no new write arrived during the previous
+      // pass. The loop re-checks AFTER the awaited embeddings step, so writes
+      // that land during embedding are not lost.
+      while (dirty.has(root)) {
+        dirty.delete(root);
+        rebuildMetadataLight(paths);
+
+        // Refresh embeddings only after metadata is consistent. Stale-aware and
+        // a no-op unless an embedder is configured.
+        runtime.ensureConfig(root);
+        const embedder = resolveEmbedder(runtime.config);
+        if (embedder) await reindexEmbeddings(paths, embedder);
+      }
+    } finally {
+      inflight.delete(root);
+    }
+  });
+
+  inflight.set(root, pass);
+  return pass;
+}
+
+/** Test-only: clear coalescing state between cases. */
+export function __resetIndexingState(): void {
+  inflight.clear();
+  dirty.clear();
+}

package/extensions/llm-wiki/lib/ingest-worker.ts

@@ -0,0 +1,281 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { AgentTool } from "@mariozechner/pi-agent-core";
+import type { Api, Model } from "@mariozechner/pi-ai";
+import { Type } from "typebox";
+import type { Static } from "typebox";
+import { appendEvent, rebuildMetadataLight } from "./metadata.js";
+import { runSubAgent } from "./subagent.js";
+import { type VaultPaths, fmtDate, slugify } from "./utils.js";
+
+/**
+ * Background ingest synthesis (issue #65, part of epic #63).
+ *
+ * Moves the work the main agent used to do during `wiki_ingest` — reading a
+ * captured source's extracted text and writing the source page + entity /
+ * concept pages — onto a background sub-agent, so capturing/ingesting never
+ * stalls the user.
+ *
+ * Design: the sub-agent produces ONE structured `commit_synthesis` call; the
+ * persistence (`commitSynthesis`) is fully deterministic and unit-testable
+ * without an LLM. This mirrors pi-observational-memory's single-structured-tool
+ * pattern and keeps the file-writing logic verifiable in isolation.
+ */
+
+// ── structured synthesis schema ───────────────────────────
+export const CommitSynthesisSchema = Type.Object({
+  summary: Type.String({
+    minLength: 1,
+    description: "2-3 paragraph summary of the source's key content.",
+  }),
+  key_takeaways: Type.Array(Type.String({ minLength: 1 }), {
+    description: "The most important points, one per item.",
+  }),
+  entities: Type.Array(
+    Type.Object({
+      title: Type.String({
+        minLength: 1,
+        description: "Entity name (person, org, tool, product).",
+      }),
+      description: Type.String({ description: "One-line description of the entity." }),
+    }),
+    { description: "Named entities mentioned in the source." },
+  ),
+  concepts: Type.Array(
+    Type.Object({
+      title: Type.String({ minLength: 1, description: "Concept name (idea, pattern, framework)." }),
+      definition: Type.String({ description: "One-line definition of the concept." }),
+    }),
+    { description: "Concepts discussed in the source." },
+  ),
+  quotes: Type.Optional(
+    Type.Array(
+      Type.Object({
+        text: Type.String({ minLength: 1 }),
+        attribution: Type.Optional(Type.String()),
+      }),
+      { description: "Notable verbatim quotes." },
+    ),
+  ),
+  contradictions: Type.Optional(
+    Type.Array(Type.String({ minLength: 1 }), {
+      description: "Tensions/contradictions with existing wiki content, if any.",
+    }),
+  ),
+});
+
+export type SynthesisData = Static<typeof CommitSynthesisSchema>;
+
+export interface CommitResult {
+  sourceId: string;
+  sourcePage: string;
+  entitiesCreated: string[];
+  conceptsCreated: string[];
+  entitiesLinked: string[];
+  conceptsLinked: string[];
+  contradictions: number;
+}
+
+// ── deterministic persistence (no LLM) ────────────────────
+
+function buildEntityPage(
+  title: string,
+  description: string,
+  date: string,
+  sourceId: string,
+): string {
+  const desc = description.trim() || "One-line description.";
+  return `---\ntype: entity\ncreated: ${date}\nupdated: ${date}\nsources: [[[sources/${sourceId}]]]\n---\n\n# ${title}\n\n${desc}\n\n## Overview\n\n[Key facts]\n\n## Links\n\n- [[sources/${sourceId}]]\n`;
+}
+
+function buildConceptPage(
+  title: string,
+  definition: string,
+  date: string,
+  sourceId: string,
+): string {
+  const def = definition.trim() || "One-line definition.";
+  return `---\ntype: concept\ncreated: ${date}\nupdated: ${date}\nsources: [[[sources/${sourceId}]]]\n---\n\n# ${title}\n\n${def}\n\n## Definition\n\n[Clear explanation]\n\n## Links\n\n- [[sources/${sourceId}]]\n`;
+}
+
+/** Rebuild the source page from synthesis data, marking it ingested. */
+export function buildIngestedSourcePage(
+  manifest: Record<string, unknown>,
+  data: SynthesisData,
+  date: string,
+): string {
+  const id = String(manifest.id);
+  const title = String(manifest.title || id);
+  const url = manifest.url ? `\n> _Original: [${manifest.url}](${manifest.url})_` : "";
+  const format = String(manifest.format || "unknown");
+  const captured = String(manifest.captured || date);
+
+  const takeaways =
+    data.key_takeaways.length > 0
+      ? data.key_takeaways.map((t) => `- ${t.trim()}`).join("\n")
+      : "- [None recorded]";
+  const entities =
+    data.entities.length > 0
+      ? data.entities.map((e) => `- [[${slugify(e.title)}]]`).join("\n")
+      : "- [None]";
+  const concepts =
+    data.concepts.length > 0
+      ? data.concepts.map((c) => `- [[${slugify(c.title)}]]`).join("\n")
+      : "- [None]";
+  const quotes =
+    data.quotes && data.quotes.length > 0
+      ? data.quotes
+          .map((q) => `> ${q.text.trim()}${q.attribution ? ` — ${q.attribution}` : ""}`)
+          .join("\n\n")
+      : "> [None recorded]";
+  const contradictions =
+    data.contradictions && data.contradictions.length > 0
+      ? `\n## Contradictions\n\n${data.contradictions.map((c) => `⚠️ **Contradiction**: ${c.trim()}`).join("\n")}\n`
+      : "";
+
+  return `---\ntype: source\nformat: ${format}\nsource_id: ${id}\nraw_path: raw/sources/${id}/extracted.md\ncaptured: ${captured}\nstatus: ingested\nupdated: ${date}\n---\n\n# ${title}${url}\n\n## Summary\n\n${data.summary.trim()}\n\n## Key Takeaways\n\n${takeaways}\n\n## Entities Mentioned\n\n${entities}\n\n## Concepts Mentioned\n\n${concepts}\n\n## Notable Quotes\n\n${quotes}\n${contradictions}\n## Source Packet\n\n- **ID:** \`[[sources/${id}]]\`\n- **Extracted:** [raw/sources/${id}/extracted.md](../raw/sources/${id}/extracted.md)\n- **Manifest:** [raw/sources/${id}/manifest.json](../raw/sources/${id}/manifest.json)\n`;
+}
+
+/**
+ * Persist a synthesis deterministically: rewrite the source page (status →
+ * ingested), create missing entity/concept pages (existing pages are linked,
+ * never overwritten), and log the event. Pure file I/O — no LLM, no network.
+ */
+export function commitSynthesis(
+  paths: VaultPaths,
+  sourceId: string,
+  manifest: Record<string, unknown>,
+  data: SynthesisData,
+  date: string = fmtDate(),
+): CommitResult {
+  const result: CommitResult = {
+    sourceId,
+    sourcePage: join(paths.wiki, "sources", `${sourceId}.md`),
+    entitiesCreated: [],
+    conceptsCreated: [],
+    entitiesLinked: [],
+    conceptsLinked: [],
+    contradictions: data.contradictions?.length ?? 0,
+  };
+
+  // Source page (always rewritten from skeleton → ingested).
+  mkdirSync(join(paths.wiki, "sources"), { recursive: true });
+  writeFileSync(result.sourcePage, buildIngestedSourcePage(manifest, data, date), "utf-8");
+
+  // Entity pages — create if absent, link if present.
+  mkdirSync(join(paths.wiki, "entities"), { recursive: true });
+  for (const e of data.entities) {
+    const slug = slugify(e.title);
+    if (!slug) continue;
+    const pagePath = join(paths.wiki, "entities", `${slug}.md`);
+    if (existsSync(pagePath)) {
+      result.entitiesLinked.push(slug);
+    } else {
+      writeFileSync(pagePath, buildEntityPage(e.title, e.description, date, sourceId), "utf-8");
+      result.entitiesCreated.push(slug);
+    }
+  }
+
+  // Concept pages — create if absent, link if present.
+  mkdirSync(join(paths.wiki, "concepts"), { recursive: true });
+  for (const c of data.concepts) {
+    const slug = slugify(c.title);
+    if (!slug) continue;
+    const pagePath = join(paths.wiki, "concepts", `${slug}.md`);
+    if (existsSync(pagePath)) {
+      result.conceptsLinked.push(slug);
+    } else {
+      writeFileSync(pagePath, buildConceptPage(c.title, c.definition, date, sourceId), "utf-8");
+      result.conceptsCreated.push(slug);
+    }
+  }
+
+  appendEvent(paths, {
+    kind: "ingest",
+    source_id: sourceId,
+    entities_created: result.entitiesCreated.length,
+    concepts_created: result.conceptsCreated.length,
+    contradictions: result.contradictions,
+    background: true,
+  });
+
+  return result;
+}
+
+// ── sub-agent synthesis (LLM) ─────────────────────────────
+
+export const INGEST_SYSTEM = `You are the LLM Wiki ingestion synthesizer. You turn a single captured source's extracted text into structured wiki knowledge.
+
+Read the source content, then call \`commit_synthesis\` EXACTLY ONCE with:
+- summary: a faithful 2-3 paragraph summary (no fabrication).
+- key_takeaways: the most important points.
+- entities: named people, organizations, tools, products actually mentioned.
+- concepts: ideas, patterns, frameworks actually discussed.
+- quotes: notable verbatim quotes (optional).
+- contradictions: tensions with general knowledge or noted in the text (optional).
+
+Rules:
+- Never fabricate. Only include entities/concepts present in the source.
+- Keep descriptions to one line.
+- After calling commit_synthesis once, reply with a one-line confirmation and stop.`;
+
+export interface RunIngestSynthesisArgs {
+  model: Model<Api>;
+  apiKey: string;
+  headers?: Record<string, string>;
+  paths: VaultPaths;
+  sourceId: string;
+  manifest: Record<string, unknown>;
+  extracted: string;
+  /** Cap on extracted chars fed to the model (avoid huge prompts). Default 24k. */
+  maxChars?: number;
+  signal?: AbortSignal;
+}
+
+/**
+ * Run the synthesis sub-agent for a single source, then commit + rebuild
+ * metadata. Returns the commit result, or undefined if the model produced no
+ * synthesis.
+ */
+export async function runIngestSynthesis(
+  args: RunIngestSynthesisArgs,
+): Promise<CommitResult | undefined> {
+  const { model, apiKey, headers, paths, sourceId, manifest, extracted, maxChars, signal } = args;
+  const content = extracted.slice(0, maxChars ?? 24_000);
+  if (!content.trim()) return undefined;
+
+  let committed: CommitResult | undefined;
+
+  const commitTool: AgentTool<typeof CommitSynthesisSchema> = {
+    name: "commit_synthesis",
+    label: "Commit synthesis",
+    description:
+      "Persist the structured synthesis of this source into wiki pages. Call exactly once.",
+    parameters: CommitSynthesisSchema,
+    execute: async (_id, params) => {
+      committed = commitSynthesis(paths, sourceId, manifest, params);
+      const ack = `Committed: source page + ${committed.entitiesCreated.length} new entit${
+        committed.entitiesCreated.length === 1 ? "y" : "ies"
+      }, ${committed.conceptsCreated.length} new concept${
+        committed.conceptsCreated.length === 1 ? "" : "s"
+      }. Reply with a one-line confirmation and stop.`;
+      return { content: [{ type: "text", text: ack }], details: { sourceId } };
+    },
+  };
+
+  const title = String(manifest.title || sourceId);
+  const userPrompt = `Synthesize this captured source into wiki knowledge by calling commit_synthesis once.\n\nSOURCE: ${title} (${sourceId})\n\nEXTRACTED CONTENT:\n${content}`;
+
+  await runSubAgent({
+    model,
+    apiKey,
+    headers,
+    systemPrompt: INGEST_SYSTEM,
+    userPrompt,
+    tools: [commitTool as AgentTool],
+    signal,
+  });
+
+  if (committed) rebuildMetadataLight(paths);
+  return committed;
+}

package/extensions/llm-wiki/lib/model-command.ts

@@ -0,0 +1,128 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import type { Runtime } from "./runtime.js";
+import { type TaskConfig, parseModelRef, persistTaskModel } from "./task-config.js";
+
+/**
+ * Model selection surface for the wiki background lane (issue #69, epic #63).
+ *
+ * The `taskModel` config field (read by `Runtime.resolveModel`) already exists;
+ * this module adds the user-facing *surface* to view and set it:
+ *   - the `/wiki-model` slash command (interactive picker + scriptable arg),
+ *   - a status-bar label of the active task model,
+ *   - the per-call override is wired on heavy tools (e.g. `wiki_ingest`).
+ *
+ * The default is always the session model: when no `taskModel` is configured
+ * and no override is passed, background work runs on the current session model.
+ */
+
+/** A minimal view of a registry model (provider + id, optional display name). */
+interface ModelLike {
+  provider: string;
+  id: string;
+  name?: string;
+}
+
+/** Words that clear the override and revert to the session model. */
+const CLEAR_WORDS = new Set(["session", "default", "reset", "clear", "none", "unset"]);
+
+/** The status-bar key for the active-model label (so we can update it in place). */
+export const MODEL_STATUS_KEY = "llm-wiki-model";
+
+/**
+ * Human-readable label for the active background task model. Shows the
+ * configured `provider/id` when set, otherwise the session model (with its id
+ * when known). Pure — safe to unit test and reuse for the status line.
+ */
+export function formatActiveModelLabel(config: TaskConfig, sessionModelId?: string): string {
+  if (config.taskModel) return `${config.taskModel.provider}/${config.taskModel.id}`;
+  return sessionModelId ? `session model (${sessionModelId})` : "session model";
+}
+
+/** "provider/id" ref for a model. */
+function modelRef(m: ModelLike): string {
+  return `${m.provider}/${m.id}`;
+}
+
+/**
+ * Register the `/wiki-model` slash command. Lets the user view the active
+ * background task model and choose another (or revert to the session model).
+ * The choice is persisted to project settings and applied immediately.
+ *
+ *   /wiki-model                 → interactive picker (lists available models)
+ *   /wiki-model provider/id     → set directly (scriptable / no UI needed)
+ *   /wiki-model session|clear   → clear the override, use the session model
+ */
+export function registerWikiModelCommand(pi: ExtensionAPI, runtime: Runtime): void {
+  pi.registerCommand("wiki-model", {
+    description:
+      "View or set the model used for LLM Wiki background tasks (default: session model)",
+    handler: async (args, ctx) => {
+      runtime.ensureConfig(ctx.cwd);
+      const sessionId = (ctx.model as ModelLike | undefined)?.id;
+
+      const apply = (model: { provider: string; id: string } | undefined): void => {
+        persistTaskModel(ctx.cwd, model);
+        runtime.config = { ...runtime.config, taskModel: model };
+        runtime.configLoaded = true;
+        const label = formatActiveModelLabel(runtime.config, sessionId);
+        ctx.ui.setStatus(MODEL_STATUS_KEY, `🧠 wiki model: ${label}`);
+        ctx.ui.notify(`LLM Wiki: background tasks now use ${label}`, "info");
+      };
+
+      const trimmed = args.trim();
+
+      // Explicit clear → session model.
+      if (trimmed && CLEAR_WORDS.has(trimmed.toLowerCase())) {
+        apply(undefined);
+        return;
+      }
+
+      // Direct "provider/id" set (works without UI).
+      if (trimmed) {
+        const ref = parseModelRef(trimmed);
+        if (!ref) {
+          ctx.ui.notify(
+            `LLM Wiki: could not parse "${trimmed}". Use provider/id (e.g. anthropic/claude-haiku) or "session".`,
+            "error",
+          );
+          return;
+        }
+        const found = ctx.modelRegistry.find(ref.provider, ref.id) as ModelLike | undefined;
+        if (!found) {
+          ctx.ui.notify(
+            `LLM Wiki: model ${ref.provider}/${ref.id} is not in the registry (run /wiki-model with no argument to pick from available models).`,
+            "error",
+          );
+          return;
+        }
+        apply({ provider: found.provider, id: found.id });
+        return;
+      }
+
+      // No argument: interactive picker.
+      const current = formatActiveModelLabel(runtime.config, sessionId);
+      if (!ctx.hasUI) {
+        ctx.ui.notify(
+          `LLM Wiki: active background model is ${current}. Pass provider/id to change it (no interactive UI here).`,
+          "info",
+        );
+        return;
+      }
+
+      const available = (ctx.modelRegistry.getAvailable() as ModelLike[]) ?? [];
+      const pool = available.length > 0 ? available : (ctx.modelRegistry.getAll() as ModelLike[]);
+      const sessionOption = "↩ Use session model (clear override)";
+      const options = [sessionOption, ...pool.map(modelRef)];
+
+      const picked = await ctx.ui.select(`Wiki background model (current: ${current})`, options);
+      if (picked === undefined) return; // cancelled
+
+      if (picked === sessionOption) {
+        apply(undefined);
+        return;
+      }
+      const ref = parseModelRef(picked);
+      if (ref) apply(ref);
+    },
+  });
+}

package/extensions/llm-wiki/lib/observation.ts

@@ -2,7 +2,9 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
 import { Type } from "typebox";
+import { scheduleReindex } from "./indexing.js";
 import { appendEvent, rebuildMetadataLight } from "./metadata.js";
+import type { Runtime } from "./runtime.js";
 import { type VaultPaths, fmtDate, resolveVaultPaths } from "./utils.js";
 
 // ─── Types ─────────────────────────────────────────────
@@ -44,7 +46,11 @@ const RELEVANCE_EMOJIS: Record<string, string> = {
  * Observations are stored in wiki/sources/ with type: source and
  * status: observation. They are searchable via wiki_recail.
  */
-export function saveObservation(paths: VaultPaths, input: ObservationInput): ObservationResult {
+export function saveObservation(
+  paths: VaultPaths,
+  input: ObservationInput,
+  opts?: { rebuild?: boolean },
+): ObservationResult {
   const today = fmtDate();
   const timestamp = new Date().toISOString();
 
@@ -109,8 +115,10 @@ export function saveObservation(paths: VaultPaths, input: ObservationInput): Obs
     relevance: input.relevance,
   });
 
-  // Rebuild metadata so the observation is immediately searchable
-  rebuildMetadataLight(paths);
+  // Rebuild metadata so the observation is immediately searchable. Callers that
+  // background this (the wiki_observe tool) pass { rebuild: false } and schedule
+  // a non-blocking reindex instead.
+  if (opts?.rebuild !== false) rebuildMetadataLight(paths);
 
   return { slug, pagePath };
 }
@@ -137,7 +145,11 @@ export function createReminderState(): ReminderState {
  * The model calls this to record observations during a session.
  * Observations are saved to the wiki and become searchable.
  */
-export function registerWikiObserve(pi: ExtensionAPI, reminderState?: ReminderState): void {
+export function registerWikiObserve(
+  pi: ExtensionAPI,
+  runtime?: Runtime,
+  reminderState?: ReminderState,
+): void {
   pi.registerTool({
     name: "wiki_observe",
     label: "Wiki Observe",
@@ -214,13 +226,24 @@ export function registerWikiObserve(pi: ExtensionAPI, reminderState?: ReminderSt
         };
       }
 
-      const result = saveObservation(paths, {
-        title: params.title,
-        content: params.content,
-        relevance: params.relevance,
-        tags: params.tags,
-        source_context: params.source_context,
-      });
+      const result = saveObservation(
+        paths,
+        {
+          title: params.title,
+          content: params.content,
+          relevance: params.relevance,
+          tags: params.tags,
+          source_context: params.source_context,
+        },
+        // When a background runtime is available, write the page synchronously
+        // but defer the O(pages) metadata rebuild + embeddings off the tool's
+        // critical path. Without a runtime, fall back to the inline rebuild.
+        { rebuild: !runtime },
+      );
+      if (runtime) {
+        const launchCtx = { hasUI: ctx.hasUI, ui: ctx.ui };
+        scheduleReindex(runtime, launchCtx, paths);
+      }
 
       // Signal the reminder to stop nagging this session
       if (reminderState) {