@zosmaai/pi-llm-wiki 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +33 -3
- package/extensions/llm-wiki/index.ts +49 -10
- package/extensions/llm-wiki/lib/embeddings.ts +420 -0
- package/extensions/llm-wiki/lib/guardrails.ts +15 -4
- package/extensions/llm-wiki/lib/indexing.ts +88 -0
- package/extensions/llm-wiki/lib/ingest-worker.ts +281 -0
- package/extensions/llm-wiki/lib/model-command.ts +128 -0
- package/extensions/llm-wiki/lib/observation.ts +34 -11
- package/extensions/llm-wiki/lib/recall.ts +331 -10
- package/extensions/llm-wiki/lib/retro.ts +13 -4
- package/extensions/llm-wiki/lib/runtime.ts +216 -0
- package/extensions/llm-wiki/lib/source-extractors.ts +120 -11
- package/extensions/llm-wiki/lib/source-packet.ts +19 -1
- package/extensions/llm-wiki/lib/subagent.ts +82 -0
- package/extensions/llm-wiki/lib/task-config.ts +195 -0
- package/extensions/llm-wiki/lib/tools.ts +178 -8
- package/package.json +3 -2
- package/prompts/wiki-ingest.md +7 -4
- package/skills/llm-wiki/SKILL.md +30 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { getAgentDir } from "@mariozechner/pi-coding-agent";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Configuration for the background-task lane (issue #64, part of #63).
|
|
7
|
+
*
|
|
8
|
+
* The wiki's intelligent work (ingest synthesis, embeddings, topic inference)
|
|
9
|
+
* can run off the main agent thread on a model of the user's choosing. This
|
|
10
|
+
* module resolves that configuration from pi's namespaced settings, mirroring
|
|
11
|
+
* the approach used by pi-observational-memory.
|
|
12
|
+
*
|
|
13
|
+
* Resolution order (later wins):
|
|
14
|
+
* 1. built-in DEFAULTS
|
|
15
|
+
* 2. global settings: <agentDir>/settings.json → { "llm-wiki": { ... } }
|
|
16
|
+
* 3. project settings: <cwd>/.pi/settings.json → { "llm-wiki": { ... } }
|
|
17
|
+
*
|
|
18
|
+
* When `taskModel` is unset, the background lane falls back to the session
|
|
19
|
+
* model (see Runtime.resolveModel), so the feature is zero-config by default.
|
|
20
|
+
*/
|
|
21
|
+
export interface TaskConfig {
|
|
22
|
+
/**
|
|
23
|
+
* Model used for background wiki tasks. When undefined, the session model
|
|
24
|
+
* is used. The surface for setting this (config field, /command, per-call
|
|
25
|
+
* override) is built in issue #69; this module only reads it.
|
|
26
|
+
*/
|
|
27
|
+
taskModel?: { provider: string; id: string };
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Embedding provider for background write-time embeddings (issue #66).
|
|
31
|
+
* Only "openai" / "openai-compatible" are supported. When undefined,
|
|
32
|
+
* embeddings are disabled entirely (silent no-op) — this is the default,
|
|
33
|
+
* so the feature is strictly opt-in.
|
|
34
|
+
*/
|
|
35
|
+
embeddingProvider?: string;
|
|
36
|
+
/** Embedding model id (default: text-embedding-3-small). */
|
|
37
|
+
embeddingModel?: string;
|
|
38
|
+
/** OpenAI-compatible base URL (default: https://api.openai.com or OPENAI_BASE_URL). */
|
|
39
|
+
embeddingBaseUrl?: string;
|
|
40
|
+
/**
|
|
41
|
+
* Embedding API key. Prefer `embeddingApiKeyEnv` to avoid storing secrets in
|
|
42
|
+
* settings files; this direct field exists for parity but is discouraged.
|
|
43
|
+
*/
|
|
44
|
+
embeddingApiKey?: string;
|
|
45
|
+
/** Env var name to read the embedding API key from (default: OPENAI_API_KEY). */
|
|
46
|
+
embeddingApiKeyEnv?: string;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Weight of the semantic (cosine) signal when blending with lexical score in
|
|
50
|
+
* hybrid recall (issue #67). 0 = pure lexical, 1 = pure semantic boost.
|
|
51
|
+
* Default 0.5. Only takes effect when embeddings exist AND an embedder is
|
|
52
|
+
* configured; otherwise recall stays 100% lexical.
|
|
53
|
+
*/
|
|
54
|
+
semanticWeight?: number;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Two-stage recall gate (issue #68). When the vault's registered page count
|
|
58
|
+
* is STRICTLY GREATER THAN this threshold, recall switches to "links-first"
|
|
59
|
+
* mode: it returns a ranked list of links (id, title, type, score, 1-line
|
|
60
|
+
* snippet) instead of inline content previews, and the agent expands chosen
|
|
61
|
+
* links on demand via `read`. At or below the threshold, the current
|
|
62
|
+
* preview-inline behavior is preserved (no regression for small vaults).
|
|
63
|
+
*
|
|
64
|
+
* Page-count (not token-budget) was chosen deliberately: it is derived from
|
|
65
|
+
* `meta/registry.json` in O(1) with zero extra file I/O, so the gate itself
|
|
66
|
+
* never reads page bodies — token estimation would require touching every
|
|
67
|
+
* page, defeating the "cheap recall" goal. Default 50. Set to 0 to force
|
|
68
|
+
* links-first for any non-empty vault, or a very large number to always keep
|
|
69
|
+
* previews inline. Clamped to a non-negative integer.
|
|
70
|
+
*/
|
|
71
|
+
recallLinksThreshold?: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export const TASK_DEFAULTS: TaskConfig = {};
|
|
75
|
+
|
|
76
|
+
const SETTINGS_KEY = "llm-wiki";
|
|
77
|
+
|
|
78
|
+
function readModelSpec(value: unknown): { provider: string; id: string } | undefined {
|
|
79
|
+
if (!value || typeof value !== "object") return undefined;
|
|
80
|
+
const v = value as Record<string, unknown>;
|
|
81
|
+
if (typeof v.provider === "string" && typeof v.id === "string" && v.provider && v.id) {
|
|
82
|
+
return { provider: v.provider, id: v.id };
|
|
83
|
+
}
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function readNamespacedConfig(path: string): Partial<TaskConfig> {
|
|
88
|
+
if (!existsSync(path)) return {};
|
|
89
|
+
try {
|
|
90
|
+
const raw = JSON.parse(readFileSync(path, "utf-8")) as Record<string, unknown>;
|
|
91
|
+
const nested = raw[SETTINGS_KEY];
|
|
92
|
+
if (!nested || typeof nested !== "object") return {};
|
|
93
|
+
const section = nested as Record<string, unknown>;
|
|
94
|
+
const out: Partial<TaskConfig> = {};
|
|
95
|
+
const taskModel = readModelSpec(section.taskModel);
|
|
96
|
+
if (taskModel) out.taskModel = taskModel;
|
|
97
|
+
|
|
98
|
+
for (const key of [
|
|
99
|
+
"embeddingProvider",
|
|
100
|
+
"embeddingModel",
|
|
101
|
+
"embeddingBaseUrl",
|
|
102
|
+
"embeddingApiKey",
|
|
103
|
+
"embeddingApiKeyEnv",
|
|
104
|
+
] as const) {
|
|
105
|
+
const value = section[key];
|
|
106
|
+
if (typeof value === "string" && value.trim()) out[key] = value.trim();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const weight = section.semanticWeight;
|
|
110
|
+
if (typeof weight === "number" && Number.isFinite(weight)) {
|
|
111
|
+
out.semanticWeight = Math.min(1, Math.max(0, weight));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const threshold = section.recallLinksThreshold;
|
|
115
|
+
if (typeof threshold === "number" && Number.isFinite(threshold)) {
|
|
116
|
+
out.recallLinksThreshold = Math.max(0, Math.floor(threshold));
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
} catch {
|
|
120
|
+
return {};
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Parse a `"provider/id"` model reference (issue #69). Splits on the FIRST
|
|
126
|
+
* slash so model ids that themselves contain slashes (e.g.
|
|
127
|
+
* `openrouter/meta/llama-3`) are preserved. Returns `undefined` for empty,
|
|
128
|
+
* slashless, or partial (`provider/` / `/id`) refs so callers can reject bad
|
|
129
|
+
* input. Whitespace is trimmed.
|
|
130
|
+
*/
|
|
131
|
+
export function parseModelRef(ref: string): { provider: string; id: string } | undefined {
|
|
132
|
+
const trimmed = ref.trim();
|
|
133
|
+
const slash = trimmed.indexOf("/");
|
|
134
|
+
if (slash <= 0) return undefined;
|
|
135
|
+
const provider = trimmed.slice(0, slash).trim();
|
|
136
|
+
const id = trimmed.slice(slash + 1).trim();
|
|
137
|
+
if (!provider || !id) return undefined;
|
|
138
|
+
return { provider, id };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Persist (or clear) the wiki background `taskModel` in the PROJECT settings
|
|
143
|
+
* file `<cwd>/.pi/settings.json` under the namespaced `llm-wiki` key (issue
|
|
144
|
+
* #69). Project settings win over global in `loadTaskConfig`, so this takes
|
|
145
|
+
* effect immediately on the next config load. Other top-level keys and other
|
|
146
|
+
* `llm-wiki` settings are preserved; passing `undefined` removes the key
|
|
147
|
+
* (reverting to the session model).
|
|
148
|
+
*/
|
|
149
|
+
export function persistTaskModel(
|
|
150
|
+
cwd: string,
|
|
151
|
+
model: { provider: string; id: string } | undefined,
|
|
152
|
+
): void {
|
|
153
|
+
const settingsPath = join(cwd, ".pi", "settings.json");
|
|
154
|
+
let raw: Record<string, unknown> = {};
|
|
155
|
+
if (existsSync(settingsPath)) {
|
|
156
|
+
try {
|
|
157
|
+
const parsed = JSON.parse(readFileSync(settingsPath, "utf-8"));
|
|
158
|
+
if (parsed && typeof parsed === "object") raw = parsed as Record<string, unknown>;
|
|
159
|
+
} catch {
|
|
160
|
+
// Corrupt settings file: start from an empty object rather than throw.
|
|
161
|
+
raw = {};
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const existing = raw[SETTINGS_KEY];
|
|
166
|
+
const section: Record<string, unknown> =
|
|
167
|
+
existing && typeof existing === "object" ? { ...(existing as Record<string, unknown>) } : {};
|
|
168
|
+
|
|
169
|
+
if (model) {
|
|
170
|
+
section.taskModel = { provider: model.provider, id: model.id };
|
|
171
|
+
} else {
|
|
172
|
+
// biome-ignore lint/performance/noDelete: one-off settings rewrite, not a hot path; removing the key (vs setting undefined) keeps the JSON clean
|
|
173
|
+
delete section.taskModel;
|
|
174
|
+
}
|
|
175
|
+
raw[SETTINGS_KEY] = section;
|
|
176
|
+
|
|
177
|
+
mkdirSync(dirname(settingsPath), { recursive: true });
|
|
178
|
+
writeFileSync(settingsPath, `${JSON.stringify(raw, null, 2)}\n`, "utf-8");
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export function loadTaskConfig(cwd: string): TaskConfig {
|
|
182
|
+
let globalPath: string;
|
|
183
|
+
try {
|
|
184
|
+
globalPath = join(getAgentDir(), "settings.json");
|
|
185
|
+
} catch {
|
|
186
|
+
globalPath = "";
|
|
187
|
+
}
|
|
188
|
+
const projectPath = join(cwd, ".pi", "settings.json");
|
|
189
|
+
|
|
190
|
+
return {
|
|
191
|
+
...TASK_DEFAULTS,
|
|
192
|
+
...(globalPath ? readNamespacedConfig(globalPath) : {}),
|
|
193
|
+
...readNamespacedConfig(projectPath),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
@@ -2,6 +2,9 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
4
4
|
import { Type } from "typebox";
|
|
5
|
+
import { launchEmbedPages, reindexEmbeddings, resolveEmbedder } from "./embeddings.js";
|
|
6
|
+
import { scheduleReindex } from "./indexing.js";
|
|
7
|
+
import { runIngestSynthesis } from "./ingest-worker.js";
|
|
5
8
|
import {
|
|
6
9
|
type Registry,
|
|
7
10
|
appendEvent,
|
|
@@ -10,7 +13,9 @@ import {
|
|
|
10
13
|
rebuildMetadata,
|
|
11
14
|
rebuildMetadataLight,
|
|
12
15
|
} from "./metadata.js";
|
|
16
|
+
import type { Runtime } from "./runtime.js";
|
|
13
17
|
import { captureFile, captureText, captureUrl } from "./source-packet.js";
|
|
18
|
+
import { parseModelRef } from "./task-config.js";
|
|
14
19
|
import {
|
|
15
20
|
type VaultPaths,
|
|
16
21
|
detectVaultFormat,
|
|
@@ -142,7 +147,7 @@ export function registerWikiBootstrap(pi: ExtensionAPI): void {
|
|
|
142
147
|
|
|
143
148
|
// ─── 2. wiki_capture_source ─────────────────────────────
|
|
144
149
|
|
|
145
|
-
export function registerWikiCaptureSource(pi: ExtensionAPI): void {
|
|
150
|
+
export function registerWikiCaptureSource(pi: ExtensionAPI, runtime?: Runtime): void {
|
|
146
151
|
pi.registerTool({
|
|
147
152
|
name: "wiki_capture_source",
|
|
148
153
|
label: "Wiki Capture Source",
|
|
@@ -191,7 +196,11 @@ export function registerWikiCaptureSource(pi: ExtensionAPI): void {
|
|
|
191
196
|
};
|
|
192
197
|
}
|
|
193
198
|
|
|
194
|
-
|
|
199
|
+
if (runtime) {
|
|
200
|
+
scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
|
|
201
|
+
} else {
|
|
202
|
+
rebuildMetadataLight(paths);
|
|
203
|
+
}
|
|
195
204
|
|
|
196
205
|
return {
|
|
197
206
|
content: [
|
|
@@ -220,16 +229,17 @@ export function registerWikiCaptureSource(pi: ExtensionAPI): void {
|
|
|
220
229
|
|
|
221
230
|
// ─── 3. wiki_ingest ─────────────────────────────────────
|
|
222
231
|
|
|
223
|
-
export function registerWikiIngest(pi: ExtensionAPI): void {
|
|
232
|
+
export function registerWikiIngest(pi: ExtensionAPI, runtime?: Runtime): void {
|
|
224
233
|
pi.registerTool({
|
|
225
234
|
name: "wiki_ingest",
|
|
226
235
|
label: "Wiki Ingest",
|
|
227
236
|
description:
|
|
228
|
-
"Process uningested source packets.
|
|
229
|
-
promptSnippet: "Ingest source packets
|
|
237
|
+
"Process uningested source packets. By default synthesis runs in the background (non-blocking) on the configured task model; pass background=false to return extracted content for the main agent to synthesize itself.",
|
|
238
|
+
promptSnippet: "Ingest source packets (background synthesis by default)",
|
|
230
239
|
promptGuidelines: [
|
|
231
240
|
"Use wiki_ingest when the user wants to process captured sources.",
|
|
232
|
-
"
|
|
241
|
+
"By default ingestion runs in the BACKGROUND — you'll get a notification, not extracted content. Do NOT synthesize those sources yourself.",
|
|
242
|
+
"If the tool returns extracted content (background unavailable, or background=false), then read each source's extracted.md, update its source page, create entity/concept pages, and cross-reference.",
|
|
233
243
|
"The extension auto-updates metadata — you do NOT need to edit meta/ files.",
|
|
234
244
|
],
|
|
235
245
|
parameters: Type.Object({
|
|
@@ -237,7 +247,20 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
|
|
|
237
247
|
Type.String({ description: "Specific source ID to ingest. Leave empty for all new." }),
|
|
238
248
|
),
|
|
239
249
|
batch_size: Type.Optional(
|
|
240
|
-
Type.Number({ description: "Max sources to
|
|
250
|
+
Type.Number({ description: "Max sources to process (default: 3, max: 5)", default: 3 }),
|
|
251
|
+
),
|
|
252
|
+
background: Type.Optional(
|
|
253
|
+
Type.Boolean({
|
|
254
|
+
description:
|
|
255
|
+
"Synthesize in the background without blocking (default: true). Set false to return extracted content for the main agent to synthesize.",
|
|
256
|
+
default: true,
|
|
257
|
+
}),
|
|
258
|
+
),
|
|
259
|
+
model: Type.Optional(
|
|
260
|
+
Type.String({
|
|
261
|
+
description:
|
|
262
|
+
"Per-call model override as 'provider/id' (e.g. anthropic/claude-haiku). Overrides the configured wiki taskModel for this call; defaults to the configured/session model.",
|
|
263
|
+
}),
|
|
241
264
|
),
|
|
242
265
|
}),
|
|
243
266
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
@@ -321,6 +344,75 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
|
|
|
321
344
|
return { id, extracted, manifest };
|
|
322
345
|
});
|
|
323
346
|
|
|
347
|
+
// ── Background synthesis (issue #65) ──────────────────
|
|
348
|
+
// Default path: dispatch each source to a background sub-agent so the
|
|
349
|
+
// main agent is not blocked. Falls back to the synchronous return below
|
|
350
|
+
// when no runtime/model is available (resolveModel ok:false).
|
|
351
|
+
const wantBackground = params.background !== false;
|
|
352
|
+
if (wantBackground && runtime) {
|
|
353
|
+
runtime.ensureConfig(ctx.cwd);
|
|
354
|
+
// Per-call model override (issue #69): 'provider/id' beats the
|
|
355
|
+
// configured taskModel; a malformed/unknown ref degrades to the
|
|
356
|
+
// configured/session model inside resolveModel.
|
|
357
|
+
const override = params.model ? parseModelRef(params.model) : undefined;
|
|
358
|
+
const resolved = await runtime.resolveModel(ctx, override);
|
|
359
|
+
if (resolved.ok) {
|
|
360
|
+
const launchCtx = { hasUI: ctx.hasUI, ui: ctx.ui };
|
|
361
|
+
for (const s of sources) {
|
|
362
|
+
runtime.launchTask(launchCtx, `ingest:${s.id}`, async () => {
|
|
363
|
+
const committed = await runIngestSynthesis({
|
|
364
|
+
model: resolved.model as Parameters<typeof runIngestSynthesis>[0]["model"],
|
|
365
|
+
apiKey: resolved.apiKey,
|
|
366
|
+
headers: resolved.headers,
|
|
367
|
+
paths,
|
|
368
|
+
sourceId: s.id,
|
|
369
|
+
manifest: s.manifest,
|
|
370
|
+
extracted: s.extracted,
|
|
371
|
+
});
|
|
372
|
+
if (committed) {
|
|
373
|
+
// Background semantic embeddings (#66): embed the pages this
|
|
374
|
+
// ingest just wrote, off-thread. No-op when unconfigured.
|
|
375
|
+
const pageIds = [
|
|
376
|
+
`sources/${committed.sourceId}`,
|
|
377
|
+
...committed.entitiesCreated.map((e) => `entities/${e}`),
|
|
378
|
+
...committed.entitiesLinked.map((e) => `entities/${e}`),
|
|
379
|
+
...committed.conceptsCreated.map((c) => `concepts/${c}`),
|
|
380
|
+
...committed.conceptsLinked.map((c) => `concepts/${c}`),
|
|
381
|
+
];
|
|
382
|
+
launchEmbedPages(runtime, launchCtx, paths, pageIds, `embed:ingest:${s.id}`);
|
|
383
|
+
}
|
|
384
|
+
if (ctx.hasUI) {
|
|
385
|
+
ctx.ui.notify(
|
|
386
|
+
committed
|
|
387
|
+
? `LLM Wiki: ingested ${s.id} → ${committed.entitiesCreated.length} entit${committed.entitiesCreated.length === 1 ? "y" : "ies"}, ${committed.conceptsCreated.length} concept${committed.conceptsCreated.length === 1 ? "" : "s"}`
|
|
388
|
+
: `LLM Wiki: ${s.id} produced no synthesis`,
|
|
389
|
+
committed ? "info" : "warning",
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
return {
|
|
395
|
+
content: [
|
|
396
|
+
{
|
|
397
|
+
type: "text",
|
|
398
|
+
text: [
|
|
399
|
+
`🔄 **Ingesting ${sources.length} source(s) in the background** (${toProcess.length - batch.length} remaining).`,
|
|
400
|
+
"",
|
|
401
|
+
...sources.map((s) => `- **${s.id}**: ${s.manifest.title || s.id}`),
|
|
402
|
+
"",
|
|
403
|
+
"Synthesis runs on the configured task model without blocking. You'll be notified as each source completes — do NOT synthesize these yourself.",
|
|
404
|
+
].join("\n"),
|
|
405
|
+
},
|
|
406
|
+
],
|
|
407
|
+
details: {
|
|
408
|
+
background: true,
|
|
409
|
+
dispatched: sources.map((s) => s.id),
|
|
410
|
+
remaining: toProcess.length - batch.length,
|
|
411
|
+
} as Record<string, unknown>,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
324
416
|
return {
|
|
325
417
|
content: [
|
|
326
418
|
{
|
|
@@ -359,7 +451,7 @@ export function registerWikiIngest(pi: ExtensionAPI): void {
|
|
|
359
451
|
|
|
360
452
|
// ─── 4. wiki_ensure_page ────────────────────────────────
|
|
361
453
|
|
|
362
|
-
export function registerWikiEnsurePage(pi: ExtensionAPI): void {
|
|
454
|
+
export function registerWikiEnsurePage(pi: ExtensionAPI, runtime?: Runtime): void {
|
|
363
455
|
pi.registerTool({
|
|
364
456
|
name: "wiki_ensure_page",
|
|
365
457
|
label: "Wiki Ensure Page",
|
|
@@ -426,6 +518,15 @@ export function registerWikiEnsurePage(pi: ExtensionAPI): void {
|
|
|
426
518
|
path: `${folder}/${slug}`,
|
|
427
519
|
});
|
|
428
520
|
|
|
521
|
+
// Register the new page so retrieval + embeddings can see it. When a
|
|
522
|
+
// background runtime is available, the rebuild + embeddings run off the
|
|
523
|
+
// tool's critical path; otherwise fall back to a synchronous rebuild.
|
|
524
|
+
if (runtime) {
|
|
525
|
+
scheduleReindex(runtime, { hasUI: ctx.hasUI, ui: ctx.ui }, paths);
|
|
526
|
+
} else {
|
|
527
|
+
rebuildMetadataLight(paths);
|
|
528
|
+
}
|
|
529
|
+
|
|
429
530
|
return {
|
|
430
531
|
content: [{ type: "text", text: `✅ Created ${type} page: \`${pagePath}\`` }],
|
|
431
532
|
details: { path: pagePath, created: true } as Record<string, unknown>,
|
|
@@ -854,6 +955,75 @@ export function registerWikiRebuildMeta(pi: ExtensionAPI): void {
|
|
|
854
955
|
|
|
855
956
|
// ─── 9. wiki_log_event ──────────────────────────────────
|
|
856
957
|
|
|
958
|
+
export function registerWikiReindexEmbeddings(pi: ExtensionAPI, runtime?: Runtime): void {
|
|
959
|
+
pi.registerTool({
|
|
960
|
+
name: "wiki_reindex_embeddings",
|
|
961
|
+
label: "Wiki Reindex Embeddings",
|
|
962
|
+
description:
|
|
963
|
+
"Backfill / refresh semantic embeddings for the vault. Embeds pages that " +
|
|
964
|
+
"are new or stale (content changed); pass force to re-embed everything. " +
|
|
965
|
+
"No-op when no embedding provider is configured.",
|
|
966
|
+
promptSnippet: "Backfill semantic embeddings for the wiki",
|
|
967
|
+
promptGuidelines: [
|
|
968
|
+
"Use wiki_reindex_embeddings to embed an existing vault or refresh stale embeddings.",
|
|
969
|
+
"Embeddings are optional: this no-ops cleanly when no embedding provider is configured.",
|
|
970
|
+
],
|
|
971
|
+
parameters: Type.Object({
|
|
972
|
+
force: Type.Optional(
|
|
973
|
+
Type.Boolean({ description: "Re-embed every page, ignoring staleness (default: false)" }),
|
|
974
|
+
),
|
|
975
|
+
}),
|
|
976
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
977
|
+
const paths = getPaths(ctx.cwd);
|
|
978
|
+
const vaultCheck = requireVault(paths);
|
|
979
|
+
if (!vaultCheck.ok) {
|
|
980
|
+
return {
|
|
981
|
+
content: [{ type: "text", text: vaultCheck.reason }],
|
|
982
|
+
details: { error: vaultCheck.reason } as Record<string, unknown>,
|
|
983
|
+
isError: true,
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
if (runtime) runtime.ensureConfig(ctx.cwd ?? paths.root);
|
|
988
|
+
const embedder = runtime ? resolveEmbedder(runtime.config) : undefined;
|
|
989
|
+
if (!embedder) {
|
|
990
|
+
return {
|
|
991
|
+
content: [
|
|
992
|
+
{
|
|
993
|
+
type: "text",
|
|
994
|
+
text: 'ℹ️ No embedding provider configured — semantic embeddings are disabled. Set `llm-wiki.embeddingProvider` (e.g. "openai") in settings to enable.',
|
|
995
|
+
},
|
|
996
|
+
],
|
|
997
|
+
details: { enabled: false } as Record<string, unknown>,
|
|
998
|
+
};
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
const stats = await reindexEmbeddings(paths, embedder, { force: params.force === true });
|
|
1002
|
+
appendEvent(paths, {
|
|
1003
|
+
kind: "reindex_embeddings",
|
|
1004
|
+
embedded: stats.embedded,
|
|
1005
|
+
skipped: stats.skipped,
|
|
1006
|
+
pruned: stats.pruned,
|
|
1007
|
+
model: embedder.model,
|
|
1008
|
+
});
|
|
1009
|
+
|
|
1010
|
+
return {
|
|
1011
|
+
content: [
|
|
1012
|
+
{
|
|
1013
|
+
type: "text",
|
|
1014
|
+
text: `✅ Embeddings reindexed (${embedder.model}): ${stats.embedded} embedded, ${stats.skipped} fresh, ${stats.pruned} pruned.`,
|
|
1015
|
+
},
|
|
1016
|
+
],
|
|
1017
|
+
details: {
|
|
1018
|
+
enabled: true,
|
|
1019
|
+
...stats,
|
|
1020
|
+
model: embedder.model,
|
|
1021
|
+
} as Record<string, unknown>,
|
|
1022
|
+
};
|
|
1023
|
+
},
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
|
|
857
1027
|
export function registerWikiLogEvent(pi: ExtensionAPI): void {
|
|
858
1028
|
pi.registerTool({
|
|
859
1029
|
name: "wiki_log_event",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zosmaai/pi-llm-wiki",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi",
|
|
@@ -76,7 +76,8 @@
|
|
|
76
76
|
"node": ">=18"
|
|
77
77
|
},
|
|
78
78
|
"dependencies": {
|
|
79
|
-
"@modelcontextprotocol/server": "^2.0.0-alpha.2"
|
|
79
|
+
"@modelcontextprotocol/server": "^2.0.0-alpha.2",
|
|
80
|
+
"node-html-markdown": "^2.0.0"
|
|
80
81
|
},
|
|
81
82
|
"devDependencies": {
|
|
82
83
|
"@biomejs/biome": "^1.9.4",
|
package/prompts/wiki-ingest.md
CHANGED
|
@@ -15,17 +15,20 @@ $ARGUMENTS
|
|
|
15
15
|
|
|
16
16
|
## Steps
|
|
17
17
|
|
|
18
|
-
1. Call `wiki_ingest(source_id=<id if provided>, batch_size=3)
|
|
18
|
+
1. Call `wiki_ingest(source_id=<id if provided>, batch_size=3)`.
|
|
19
19
|
2. If the tool reports "All sources ingested", inform the user and stop.
|
|
20
|
-
3.
|
|
20
|
+
3. **If the tool reports it is ingesting in the background**, the synthesis sub-agent is handling those sources on the configured task model. Do NOT synthesize them yourself — just report which sources were dispatched and stop. (You'll be notified as each completes.)
|
|
21
|
+
4. **Otherwise** (the tool returned extracted content — background unavailable or `background=false`), for each source in the returned batch:
|
|
21
22
|
a. Read the extracted text from `raw/sources/<SOURCE_ID>/extracted.md`
|
|
22
23
|
b. Update the skeleton source page in `wiki/sources/` with a proper summary, key entities, and concepts
|
|
23
24
|
c. Use `wiki_ensure_page(type=entity, title=<name>)` for each new entity (people, orgs, tools, products)
|
|
24
25
|
d. Use `wiki_ensure_page(type=concept, title=<name>)` for each new concept (ideas, patterns, frameworks)
|
|
25
26
|
e. Add `[[wikilinks]]` cross-references between related pages
|
|
26
27
|
f. Flag any contradictions with existing wiki content using `⚠️ **Contradiction**` markers
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
5. After processing a synchronous batch, call `wiki_rebuild_meta` to update metadata.
|
|
29
|
+
6. Report: "Ingested [N] sources → [M] pages created/updated. [X] contradictions flagged."
|
|
30
|
+
|
|
31
|
+
> **Background vs synchronous:** ingestion runs in the background by default (non-blocking) when a task model is available, so the main agent is never stalled. It falls back to the synchronous main-agent flow above when no model/API key is configured, or when called with `background=false`.
|
|
29
32
|
|
|
30
33
|
**Rules:**
|
|
31
34
|
- Never modify files in `raw/` — source packets are immutable after capture.
|
package/skills/llm-wiki/SKILL.md
CHANGED
|
@@ -73,6 +73,20 @@ This searches both your **personal wiki** (`~/.llm-wiki/`) and the **project wik
|
|
|
73
73
|
|
|
74
74
|
The extension also briefly searches automatically, but explicit calls with task-specific terms get better results.
|
|
75
75
|
|
|
76
|
+
#### Two-Stage Recall (links-first for large vaults)
|
|
77
|
+
|
|
78
|
+
Recall scales with vault size via **two-stage retrieval** (memex-style):
|
|
79
|
+
|
|
80
|
+
- **Small vaults** (page count ≤ threshold): recall returns inline **content previews** — read them directly, no extra step.
|
|
81
|
+
- **Large vaults** (page count > threshold): recall returns a **ranked list of links** only — `id`, `title`, `type`, `score`, and a 1-line snippet. **No full previews are injected**, to protect your context window.
|
|
82
|
+
|
|
83
|
+
**The two-step contract for large vaults:**
|
|
84
|
+
|
|
85
|
+
1. **Stage 1 — scan the links.** `wiki_recall` (and the auto-injected "Relevant Wiki Knowledge (links-first)" section) gives you ranked `[[id]]` links with scores and short snippets. Use the scores and snippets to pick the few pages that actually matter.
|
|
86
|
+
2. **Stage 2 — expand on demand.** Call `read` (or `wiki_read`) on the chosen link **paths** to pull their full content. Do **not** assume the snippet is the whole page — open the link before relying on its content.
|
|
87
|
+
|
|
88
|
+
The gate is the `recallLinksThreshold` setting (namespaced `llm-wiki`, default **50** pages). Page count is read from `meta/registry.json` (O(1), no page-body I/O). Set it to `0` to force links-first always, or a large number to always keep previews inline.
|
|
89
|
+
|
|
76
90
|
### At End — Save Insights with wiki_retro
|
|
77
91
|
|
|
78
92
|
After completing any meaningful task, call `wiki_retro` to save key insights:
|
|
@@ -95,6 +109,22 @@ For thorough research, also use `wiki_search` to browse the full registry:
|
|
|
95
109
|
wiki_search(query="broad topic")
|
|
96
110
|
```
|
|
97
111
|
|
|
112
|
+
### Background Model Selection (`/wiki-model`)
|
|
113
|
+
|
|
114
|
+
The wiki's background work (ingest synthesis, etc.) runs on a model you can choose. By **default it uses the current session model** — zero config.
|
|
115
|
+
|
|
116
|
+
- **View / pick interactively:** run `/wiki-model` with no argument to see the active model and pick from the available models.
|
|
117
|
+
- **Set directly (scriptable):** `/wiki-model anthropic/claude-haiku` (a `provider/id` ref).
|
|
118
|
+
- **Revert to session model:** `/wiki-model session` (also `clear`/`default`/`reset`).
|
|
119
|
+
|
|
120
|
+
The choice is persisted to project settings (`.pi/settings.json` under `llm-wiki.taskModel`) and shown in the status bar.
|
|
121
|
+
|
|
122
|
+
**Per-call override:** heavy tools accept an optional `model` param (`provider/id`) that overrides the configured model for that one call. Precedence is **override > configured `taskModel` > session model**; an unknown ref degrades gracefully to the configured/session model. Example:
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
wiki_ingest(model="anthropic/claude-haiku")
|
|
126
|
+
```
|
|
127
|
+
|
|
98
128
|
### Auto-Bootstrap (One-Time)
|
|
99
129
|
|
|
100
130
|
The extension creates the wiki vault automatically on startup. On the first turn, it injects a directive asking you to infer topic and mode, then call:
|