@zosmaai/pi-llm-wiki 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,195 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { dirname, join } from "node:path";
3
+ import { getAgentDir } from "@mariozechner/pi-coding-agent";
4
+
5
+ /**
6
+ * Configuration for the background-task lane (issue #64, part of #63).
7
+ *
8
+ * The wiki's intelligent work (ingest synthesis, embeddings, topic inference)
9
+ * can run off the main agent thread on a model of the user's choosing. This
10
+ * module resolves that configuration from pi's namespaced settings, mirroring
11
+ * the approach used by pi-observational-memory.
12
+ *
13
+ * Resolution order (later wins):
14
+ * 1. built-in DEFAULTS
15
+ * 2. global settings: <agentDir>/settings.json → { "llm-wiki": { ... } }
16
+ * 3. project settings: <cwd>/.pi/settings.json → { "llm-wiki": { ... } }
17
+ *
18
+ * When `taskModel` is unset, the background lane falls back to the session
19
+ * model (see Runtime.resolveModel), so the feature is zero-config by default.
20
+ */
21
+ export interface TaskConfig {
22
+ /**
23
+ * Model used for background wiki tasks. When undefined, the session model
24
+ * is used. The surface for setting this (config field, /command, per-call
25
+ * override) is built in issue #69; this module only reads it.
26
+ */
27
+ taskModel?: { provider: string; id: string };
28
+
29
+ /**
30
+ * Embedding provider for background write-time embeddings (issue #66).
31
+ * Only "openai" / "openai-compatible" are supported. When undefined,
32
+ * embeddings are disabled entirely (silent no-op) — this is the default,
33
+ * so the feature is strictly opt-in.
34
+ */
35
+ embeddingProvider?: string;
36
+ /** Embedding model id (default: text-embedding-3-small). */
37
+ embeddingModel?: string;
38
+ /** OpenAI-compatible base URL (default: https://api.openai.com or OPENAI_BASE_URL). */
39
+ embeddingBaseUrl?: string;
40
+ /**
41
+ * Embedding API key. Prefer `embeddingApiKeyEnv` to avoid storing secrets in
42
+ * settings files; this direct field exists for parity but is discouraged.
43
+ */
44
+ embeddingApiKey?: string;
45
+ /** Env var name to read the embedding API key from (default: OPENAI_API_KEY). */
46
+ embeddingApiKeyEnv?: string;
47
+
48
+ /**
49
+ * Weight of the semantic (cosine) signal when blending with lexical score in
50
+ * hybrid recall (issue #67). 0 = pure lexical, 1 = pure semantic boost.
51
+ * Default 0.5. Only takes effect when embeddings exist AND an embedder is
52
+ * configured; otherwise recall stays 100% lexical.
53
+ */
54
+ semanticWeight?: number;
55
+
56
+ /**
57
+ * Two-stage recall gate (issue #68). When the vault's registered page count
58
+ * is STRICTLY GREATER THAN this threshold, recall switches to "links-first"
59
+ * mode: it returns a ranked list of links (id, title, type, score, 1-line
60
+ * snippet) instead of inline content previews, and the agent expands chosen
61
+ * links on demand via `read`. At or below the threshold, the current
62
+ * preview-inline behavior is preserved (no regression for small vaults).
63
+ *
64
+ * Page-count (not token-budget) was chosen deliberately: it is derived from
65
+ * `meta/registry.json` in O(1) with zero extra file I/O, so the gate itself
66
+ * never reads page bodies — token estimation would require touching every
67
+ * page, defeating the "cheap recall" goal. Default 50. Set to 0 to force
68
+ * links-first for any non-empty vault, or a very large number to always keep
69
+ * previews inline. Clamped to a non-negative integer.
70
+ */
71
+ recallLinksThreshold?: number;
72
+ }
73
+
74
+ export const TASK_DEFAULTS: TaskConfig = {};
75
+
76
+ const SETTINGS_KEY = "llm-wiki";
77
+
78
+ function readModelSpec(value: unknown): { provider: string; id: string } | undefined {
79
+ if (!value || typeof value !== "object") return undefined;
80
+ const v = value as Record<string, unknown>;
81
+ if (typeof v.provider === "string" && typeof v.id === "string" && v.provider && v.id) {
82
+ return { provider: v.provider, id: v.id };
83
+ }
84
+ return undefined;
85
+ }
86
+
87
+ function readNamespacedConfig(path: string): Partial<TaskConfig> {
88
+ if (!existsSync(path)) return {};
89
+ try {
90
+ const raw = JSON.parse(readFileSync(path, "utf-8")) as Record<string, unknown>;
91
+ const nested = raw[SETTINGS_KEY];
92
+ if (!nested || typeof nested !== "object") return {};
93
+ const section = nested as Record<string, unknown>;
94
+ const out: Partial<TaskConfig> = {};
95
+ const taskModel = readModelSpec(section.taskModel);
96
+ if (taskModel) out.taskModel = taskModel;
97
+
98
+ for (const key of [
99
+ "embeddingProvider",
100
+ "embeddingModel",
101
+ "embeddingBaseUrl",
102
+ "embeddingApiKey",
103
+ "embeddingApiKeyEnv",
104
+ ] as const) {
105
+ const value = section[key];
106
+ if (typeof value === "string" && value.trim()) out[key] = value.trim();
107
+ }
108
+
109
+ const weight = section.semanticWeight;
110
+ if (typeof weight === "number" && Number.isFinite(weight)) {
111
+ out.semanticWeight = Math.min(1, Math.max(0, weight));
112
+ }
113
+
114
+ const threshold = section.recallLinksThreshold;
115
+ if (typeof threshold === "number" && Number.isFinite(threshold)) {
116
+ out.recallLinksThreshold = Math.max(0, Math.floor(threshold));
117
+ }
118
+ return out;
119
+ } catch {
120
+ return {};
121
+ }
122
+ }
123
+
124
+ /**
125
+ * Parse a `"provider/id"` model reference (issue #69). Splits on the FIRST
126
+ * slash so model ids that themselves contain slashes (e.g.
127
+ * `openrouter/meta/llama-3`) are preserved. Returns `undefined` for empty,
128
+ * slashless, or partial (`provider/` / `/id`) refs so callers can reject bad
129
+ * input. Whitespace is trimmed.
130
+ */
131
+ export function parseModelRef(ref: string): { provider: string; id: string } | undefined {
132
+ const trimmed = ref.trim();
133
+ const slash = trimmed.indexOf("/");
134
+ if (slash <= 0) return undefined;
135
+ const provider = trimmed.slice(0, slash).trim();
136
+ const id = trimmed.slice(slash + 1).trim();
137
+ if (!provider || !id) return undefined;
138
+ return { provider, id };
139
+ }
140
+
141
+ /**
142
+ * Persist (or clear) the wiki background `taskModel` in the PROJECT settings
143
+ * file `<cwd>/.pi/settings.json` under the namespaced `llm-wiki` key (issue
144
+ * #69). Project settings win over global in `loadTaskConfig`, so this takes
145
+ * effect immediately on the next config load. Other top-level keys and other
146
+ * `llm-wiki` settings are preserved; passing `undefined` removes the key
147
+ * (reverting to the session model).
148
+ */
149
+ export function persistTaskModel(
150
+ cwd: string,
151
+ model: { provider: string; id: string } | undefined,
152
+ ): void {
153
+ const settingsPath = join(cwd, ".pi", "settings.json");
154
+ let raw: Record<string, unknown> = {};
155
+ if (existsSync(settingsPath)) {
156
+ try {
157
+ const parsed = JSON.parse(readFileSync(settingsPath, "utf-8"));
158
+ if (parsed && typeof parsed === "object") raw = parsed as Record<string, unknown>;
159
+ } catch {
160
+ // Corrupt settings file: start from an empty object rather than throw.
161
+ raw = {};
162
+ }
163
+ }
164
+
165
+ const existing = raw[SETTINGS_KEY];
166
+ const section: Record<string, unknown> =
167
+ existing && typeof existing === "object" ? { ...(existing as Record<string, unknown>) } : {};
168
+
169
+ if (model) {
170
+ section.taskModel = { provider: model.provider, id: model.id };
171
+ } else {
172
+ // biome-ignore lint/performance/noDelete: one-off settings rewrite, not a hot path; removing the key (vs setting undefined) keeps the JSON clean
173
+ delete section.taskModel;
174
+ }
175
+ raw[SETTINGS_KEY] = section;
176
+
177
+ mkdirSync(dirname(settingsPath), { recursive: true });
178
+ writeFileSync(settingsPath, `${JSON.stringify(raw, null, 2)}\n`, "utf-8");
179
+ }
180
+
181
+ export function loadTaskConfig(cwd: string): TaskConfig {
182
+ let globalPath: string;
183
+ try {
184
+ globalPath = join(getAgentDir(), "settings.json");
185
+ } catch {
186
+ globalPath = "";
187
+ }
188
+ const projectPath = join(cwd, ".pi", "settings.json");
189
+
190
+ return {
191
+ ...TASK_DEFAULTS,
192
+ ...(globalPath ? readNamespacedConfig(globalPath) : {}),
193
+ ...readNamespacedConfig(projectPath),
194
+ };
195
+ }
@@ -2,6 +2,9 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
2
2
  import { join } from "node:path";
3
3
  import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
4
4
  import { Type } from "typebox";
5
+ import { launchEmbedPages, reindexEmbeddings, resolveEmbedder } from "./embeddings.js";
6
+ import { scheduleReindex } from "./indexing.js";
7
+ import { runIngestSynthesis } from "./ingest-worker.js";
5
8
  import {
6
9
  type Registry,
7
10
  appendEvent,
@@ -10,7 +13,9 @@ import {
10
13
  rebuildMetadata,
11
14
  rebuildMetadataLight,
12
15
  } from "./metadata.js";
16
+ import type { Runtime } from "./runtime.js";
13
17
  import { captureFile, captureText, captureUrl } from "./source-packet.js";
18
+ import { parseModelRef } from "./task-config.js";
14
19
  import {
15
20
  type VaultPaths,
16
21
  detectVaultFormat,
@@ -142,7 +147,7 @@ export function registerWikiBootstrap(pi: ExtensionAPI): void {
142
147
 
143
148
  // ─── 2. wiki_capture_source ─────────────────────────────
144
149
 
145
- export function registerWikiCaptureSource(pi: ExtensionAPI): void {
150
+ export function registerWikiCaptureSource(pi: ExtensionAPI, runtime?: Runtime): void {
146
151
  pi.registerTool({
147
152
  name: "wiki_capture_source",
148
153
  label: "Wiki Capture Source",
@@ -191,7 +196,11 @@ export function registerWikiCaptureSource(pi: ExtensionAPI): void {
191
196
  };
192
197
  }
193
198
 
194
- rebuildMetadataLight(paths);
199
+ if (runtime) {
200
+ scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
201
+ } else {
202
+ rebuildMetadataLight(paths);
203
+ }
195
204
 
196
205
  return {
197
206
  content: [
@@ -220,16 +229,17 @@ export function registerWikiCaptureSource(pi: ExtensionAPI): void {
220
229
 
221
230
  // ─── 3. wiki_ingest ─────────────────────────────────────
222
231
 
223
- export function registerWikiIngest(pi: ExtensionAPI): void {
232
+ export function registerWikiIngest(pi: ExtensionAPI, runtime?: Runtime): void {
224
233
  pi.registerTool({
225
234
  name: "wiki_ingest",
226
235
  label: "Wiki Ingest",
227
236
  description:
228
- "Process uningested source packets. Returns a batch of source IDs with extracted content for the LLM to synthesize.",
229
- promptSnippet: "Ingest source packets: get batch of sources needing synthesis",
237
+ "Process uningested source packets. By default synthesis runs in the background (non-blocking) on the configured task model; pass background=false to return extracted content for the main agent to synthesize itself.",
238
+ promptSnippet: "Ingest source packets (background synthesis by default)",
230
239
  promptGuidelines: [
231
240
  "Use wiki_ingest when the user wants to process captured sources.",
232
- "After calling this tool, read each source's extracted.md, update its source page, create entity/concept pages, and cross-reference.",
241
+ "By default ingestion runs in the BACKGROUND — you'll get a notification, not extracted content. Do NOT synthesize those sources yourself.",
242
+ "If the tool returns extracted content (background unavailable, or background=false), then read each source's extracted.md, update its source page, create entity/concept pages, and cross-reference.",
233
243
  "The extension auto-updates metadata — you do NOT need to edit meta/ files.",
234
244
  ],
235
245
  parameters: Type.Object({
@@ -237,7 +247,20 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
237
247
  Type.String({ description: "Specific source ID to ingest. Leave empty for all new." }),
238
248
  ),
239
249
  batch_size: Type.Optional(
240
- Type.Number({ description: "Max sources to return (default: 3, max: 5)", default: 3 }),
250
+ Type.Number({ description: "Max sources to process (default: 3, max: 5)", default: 3 }),
251
+ ),
252
+ background: Type.Optional(
253
+ Type.Boolean({
254
+ description:
255
+ "Synthesize in the background without blocking (default: true). Set false to return extracted content for the main agent to synthesize.",
256
+ default: true,
257
+ }),
258
+ ),
259
+ model: Type.Optional(
260
+ Type.String({
261
+ description:
262
+ "Per-call model override as 'provider/id' (e.g. anthropic/claude-haiku). Overrides the configured wiki taskModel for this call; defaults to the configured/session model.",
263
+ }),
241
264
  ),
242
265
  }),
243
266
  async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
@@ -321,6 +344,75 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
321
344
  return { id, extracted, manifest };
322
345
  });
323
346
 
347
+ // ── Background synthesis (issue #65) ──────────────────
348
+ // Default path: dispatch each source to a background sub-agent so the
349
+ // main agent is not blocked. Falls back to the synchronous return below
350
+ // when no runtime/model is available (resolveModel ok:false).
351
+ const wantBackground = params.background !== false;
352
+ if (wantBackground && runtime) {
353
+ runtime.ensureConfig(ctx.cwd);
354
+ // Per-call model override (issue #69): 'provider/id' beats the
355
+ // configured taskModel; a malformed/unknown ref degrades to the
356
+ // configured/session model inside resolveModel.
357
+ const override = params.model ? parseModelRef(params.model) : undefined;
358
+ const resolved = await runtime.resolveModel(ctx, override);
359
+ if (resolved.ok) {
360
+ const launchCtx = { hasUI: ctx.hasUI, ui: ctx.ui };
361
+ for (const s of sources) {
362
+ runtime.launchTask(launchCtx, `ingest:${s.id}`, async () => {
363
+ const committed = await runIngestSynthesis({
364
+ model: resolved.model as Parameters<typeof runIngestSynthesis>[0]["model"],
365
+ apiKey: resolved.apiKey,
366
+ headers: resolved.headers,
367
+ paths,
368
+ sourceId: s.id,
369
+ manifest: s.manifest,
370
+ extracted: s.extracted,
371
+ });
372
+ if (committed) {
373
+ // Background semantic embeddings (#66): embed the pages this
374
+ // ingest just wrote, off-thread. No-op when unconfigured.
375
+ const pageIds = [
376
+ `sources/${committed.sourceId}`,
377
+ ...committed.entitiesCreated.map((e) => `entities/${e}`),
378
+ ...committed.entitiesLinked.map((e) => `entities/${e}`),
379
+ ...committed.conceptsCreated.map((c) => `concepts/${c}`),
380
+ ...committed.conceptsLinked.map((c) => `concepts/${c}`),
381
+ ];
382
+ launchEmbedPages(runtime, launchCtx, paths, pageIds, `embed:ingest:${s.id}`);
383
+ }
384
+ if (ctx.hasUI) {
385
+ ctx.ui.notify(
386
+ committed
387
+ ? `LLM Wiki: ingested ${s.id} → ${committed.entitiesCreated.length} entit${committed.entitiesCreated.length === 1 ? "y" : "ies"}, ${committed.conceptsCreated.length} concept${committed.conceptsCreated.length === 1 ? "" : "s"}`
388
+ : `LLM Wiki: ${s.id} produced no synthesis`,
389
+ committed ? "info" : "warning",
390
+ );
391
+ }
392
+ });
393
+ }
394
+ return {
395
+ content: [
396
+ {
397
+ type: "text",
398
+ text: [
399
+ `🔄 **Ingesting ${sources.length} source(s) in the background** (${toProcess.length - batch.length} remaining).`,
400
+ "",
401
+ ...sources.map((s) => `- **${s.id}**: ${s.manifest.title || s.id}`),
402
+ "",
403
+ "Synthesis runs on the configured task model without blocking. You'll be notified as each source completes — do NOT synthesize these yourself.",
404
+ ].join("\n"),
405
+ },
406
+ ],
407
+ details: {
408
+ background: true,
409
+ dispatched: sources.map((s) => s.id),
410
+ remaining: toProcess.length - batch.length,
411
+ } as Record<string, unknown>,
412
+ };
413
+ }
414
+ }
415
+
324
416
  return {
325
417
  content: [
326
418
  {
@@ -359,7 +451,7 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
359
451
 
360
452
  // ─── 4. wiki_ensure_page ────────────────────────────────
361
453
 
362
- export function registerWikiEnsurePage(pi: ExtensionAPI): void {
454
+ export function registerWikiEnsurePage(pi: ExtensionAPI, runtime?: Runtime): void {
363
455
  pi.registerTool({
364
456
  name: "wiki_ensure_page",
365
457
  label: "Wiki Ensure Page",
@@ -426,6 +518,15 @@ export function registerWikiEnsurePage(pi: ExtensionAPI): void {
426
518
  path: `${folder}/${slug}`,
427
519
  });
428
520
 
521
+ // Register the new page so retrieval + embeddings can see it. When a
522
+ // background runtime is available, the rebuild + embeddings run off the
523
+ // tool's critical path; otherwise fall back to a synchronous rebuild.
524
+ if (runtime) {
525
+ scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
526
+ } else {
527
+ rebuildMetadataLight(paths);
528
+ }
529
+
429
530
  return {
430
531
  content: [{ type: "text", text: `✅ Created ${type} page: \`${pagePath}\`` }],
431
532
  details: { path: pagePath, created: true } as Record<string, unknown>,
@@ -854,6 +955,75 @@ export function registerWikiRebuildMeta(pi: ExtensionAPI): void {
854
955
 
855
956
  // ─── 9. wiki_log_event ──────────────────────────────────
856
957
 
958
+ export function registerWikiReindexEmbeddings(pi: ExtensionAPI, runtime?: Runtime): void {
959
+ pi.registerTool({
960
+ name: "wiki_reindex_embeddings",
961
+ label: "Wiki Reindex Embeddings",
962
+ description:
963
+ "Backfill / refresh semantic embeddings for the vault. Embeds pages that " +
964
+ "are new or stale (content changed); pass force to re-embed everything. " +
965
+ "No-op when no embedding provider is configured.",
966
+ promptSnippet: "Backfill semantic embeddings for the wiki",
967
+ promptGuidelines: [
968
+ "Use wiki_reindex_embeddings to embed an existing vault or refresh stale embeddings.",
969
+ "Embeddings are optional: this no-ops cleanly when no embedding provider is configured.",
970
+ ],
971
+ parameters: Type.Object({
972
+ force: Type.Optional(
973
+ Type.Boolean({ description: "Re-embed every page, ignoring staleness (default: false)" }),
974
+ ),
975
+ }),
976
+ async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
977
+ const paths = getPaths(ctx.cwd);
978
+ const vaultCheck = requireVault(paths);
979
+ if (!vaultCheck.ok) {
980
+ return {
981
+ content: [{ type: "text", text: vaultCheck.reason }],
982
+ details: { error: vaultCheck.reason } as Record<string, unknown>,
983
+ isError: true,
984
+ };
985
+ }
986
+
987
+ if (runtime) runtime.ensureConfig(ctx.cwd ?? paths.root);
988
+ const embedder = runtime ? resolveEmbedder(runtime.config) : undefined;
989
+ if (!embedder) {
990
+ return {
991
+ content: [
992
+ {
993
+ type: "text",
994
+ text: 'ℹ️ No embedding provider configured — semantic embeddings are disabled. Set `llm-wiki.embeddingProvider` (e.g. "openai") in settings to enable.',
995
+ },
996
+ ],
997
+ details: { enabled: false } as Record<string, unknown>,
998
+ };
999
+ }
1000
+
1001
+ const stats = await reindexEmbeddings(paths, embedder, { force: params.force === true });
1002
+ appendEvent(paths, {
1003
+ kind: "reindex_embeddings",
1004
+ embedded: stats.embedded,
1005
+ skipped: stats.skipped,
1006
+ pruned: stats.pruned,
1007
+ model: embedder.model,
1008
+ });
1009
+
1010
+ return {
1011
+ content: [
1012
+ {
1013
+ type: "text",
1014
+ text: `✅ Embeddings reindexed (${embedder.model}): ${stats.embedded} embedded, ${stats.skipped} fresh, ${stats.pruned} pruned.`,
1015
+ },
1016
+ ],
1017
+ details: {
1018
+ enabled: true,
1019
+ ...stats,
1020
+ model: embedder.model,
1021
+ } as Record<string, unknown>,
1022
+ };
1023
+ },
1024
+ });
1025
+ }
1026
+
857
1027
  export function registerWikiLogEvent(pi: ExtensionAPI): void {
858
1028
  pi.registerTool({
859
1029
  name: "wiki_log_event",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zosmaai/pi-llm-wiki",
3
- "version": "0.8.1",
3
+ "version": "0.9.0",
4
4
  "description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
5
5
  "keywords": [
6
6
  "pi",
@@ -76,7 +76,8 @@
76
76
  "node": ">=18"
77
77
  },
78
78
  "dependencies": {
79
- "@modelcontextprotocol/server": "^2.0.0-alpha.2"
79
+ "@modelcontextprotocol/server": "^2.0.0-alpha.2",
80
+ "node-html-markdown": "^2.0.0"
80
81
  },
81
82
  "devDependencies": {
82
83
  "@biomejs/biome": "^1.9.4",
@@ -15,17 +15,20 @@ $ARGUMENTS
15
15
 
16
16
  ## Steps
17
17
 
18
- 1. Call `wiki_ingest(source_id=<id if provided>, batch_size=3)` to get sources needing synthesis.
18
+ 1. Call `wiki_ingest(source_id=<id if provided>, batch_size=3)`.
19
19
  2. If the tool reports "All sources ingested", inform the user and stop.
20
- 3. For each source in the returned batch:
20
+ 3. **If the tool reports it is ingesting in the background**, the synthesis sub-agent is handling those sources on the configured task model. Do NOT synthesize them yourself — just report which sources were dispatched and stop. (You'll be notified as each completes.)
21
+ 4. **Otherwise** (the tool returned extracted content — background unavailable or `background=false`), for each source in the returned batch:
21
22
  a. Read the extracted text from `raw/sources/<SOURCE_ID>/extracted.md`
22
23
  b. Update the skeleton source page in `wiki/sources/` with a proper summary, key entities, and concepts
23
24
  c. Use `wiki_ensure_page(type=entity, title=<name>)` for each new entity (people, orgs, tools, products)
24
25
  d. Use `wiki_ensure_page(type=concept, title=<name>)` for each new concept (ideas, patterns, frameworks)
25
26
  e. Add `[[wikilinks]]` cross-references between related pages
26
27
  f. Flag any contradictions with existing wiki content using `⚠️ **Contradiction**` markers
27
- 4. After processing the batch, call `wiki_rebuild_meta` to update metadata.
28
- 5. Report: "Ingested [N] sources → [M] pages created/updated. [X] contradictions flagged."
28
+ 5. After processing a synchronous batch, call `wiki_rebuild_meta` to update metadata.
29
+ 6. Report: "Ingested [N] sources → [M] pages created/updated. [X] contradictions flagged."
30
+
31
+ > **Background vs synchronous:** ingestion runs in the background by default (non-blocking) when a task model is available, so the main agent is never stalled. It falls back to the synchronous main-agent flow above when no model/API key is configured, or when called with `background=false`.
29
32
 
30
33
  **Rules:**
31
34
  - Never modify files in `raw/` — source packets are immutable after capture.
@@ -73,6 +73,20 @@ This searches both your **personal wiki** (`~/.llm-wiki/`) and the **project wik
73
73
 
74
74
  The extension also briefly searches automatically, but explicit calls with task-specific terms get better results.
75
75
 
76
+ #### Two-Stage Recall (links-first for large vaults)
77
+
78
+ Recall scales with vault size via **two-stage retrieval** (memex-style):
79
+
80
+ - **Small vaults** (page count ≤ threshold): recall returns inline **content previews** — read them directly, no extra step.
81
+ - **Large vaults** (page count > threshold): recall returns a **ranked list of links** only — `id`, `title`, `type`, `score`, and a 1-line snippet. **No full previews are injected**, to protect your context window.
82
+
83
+ **The two-step contract for large vaults:**
84
+
85
+ 1. **Stage 1 — scan the links.** `wiki_recall` (and the auto-injected "Relevant Wiki Knowledge (links-first)" section) gives you ranked `[[id]]` links with scores and short snippets. Use the scores and snippets to pick the few pages that actually matter.
86
+ 2. **Stage 2 — expand on demand.** Call `read` (or `wiki_read`) on the chosen link **paths** to pull their full content. Do **not** assume the snippet is the whole page — open the link before relying on its content.
87
+
88
+ The gate is the `recallLinksThreshold` setting (namespaced `llm-wiki`, default **50** pages). Page count is read from `meta/registry.json` (O(1), no page-body I/O). Set it to `0` to force links-first always, or a large number to always keep previews inline.
89
+
76
90
  ### At End — Save Insights with wiki_retro
77
91
 
78
92
  After completing any meaningful task, call `wiki_retro` to save key insights:
@@ -95,6 +109,22 @@ For thorough research, also use `wiki_search` to browse the full registry:
95
109
  wiki_search(query="broad topic")
96
110
  ```
97
111
 
112
+ ### Background Model Selection (`/wiki-model`)
113
+
114
+ The wiki's background work (ingest synthesis, etc.) runs on a model you can choose. By **default it uses the current session model** — zero config.
115
+
116
+ - **View / pick interactively:** run `/wiki-model` with no argument to see the active model and pick from the available models.
117
+ - **Set directly (scriptable):** `/wiki-model anthropic/claude-haiku` (a `provider/id` ref).
118
+ - **Revert to session model:** `/wiki-model session` (also `clear`/`default`/`reset`).
119
+
120
+ The choice is persisted to project settings (`.pi/settings.json` under `llm-wiki.taskModel`) and shown in the status bar.
121
+
122
+ **Per-call override:** heavy tools accept an optional `model` param (`provider/id`) that overrides the configured model for that one call. Precedence is **override > configured `taskModel` > session model**; an unknown ref degrades gracefully to the configured/session model. Example:
123
+
124
+ ```
125
+ wiki_ingest(model="anthropic/claude-haiku")
126
+ ```
127
+
98
128
  ### Auto-Bootstrap (One-Time)
99
129
 
100
130
  The extension creates the wiki vault automatically on startup. On the first turn, it injects a directive asking you to infer topic and mode, then call: