@evandu/super-pi 0.1.84 → 0.1.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/agents/super-pi-knowledge/index.ts +40 -0
  2. package/agents/super-pi-task/index.ts +2 -2
  3. package/assets/knowledge-model/README.md +37 -0
  4. package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/config.json +31 -0
  5. package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
  6. package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/special_tokens_map.json +7 -0
  7. package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/tokenizer.json +21278 -0
  8. package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/tokenizer_config.json +15 -0
  9. package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/vocab.txt +21128 -0
  10. package/dist/.super-pi-check-inputs.json +4 -0
  11. package/dist/client/.super-pi-client-sources.json +10 -7
  12. package/dist/client/assets/index-B3QVoYJY.css +1 -0
  13. package/dist/client/assets/index-C_giUkvT.js +72 -0
  14. package/dist/client/index.html +2 -2
  15. package/dist/server/application/run-service.d.ts +50 -11
  16. package/dist/server/application/run-service.js +206 -77
  17. package/dist/server/command-handler.js +16 -4
  18. package/dist/server/http-server.js +218 -8
  19. package/dist/server/index.js +15 -1
  20. package/dist/server/knowledge-vectors.d.ts +133 -0
  21. package/dist/server/knowledge-vectors.js +302 -0
  22. package/dist/server/knowledge.d.ts +263 -0
  23. package/dist/server/knowledge.js +743 -0
  24. package/dist/server/persistence/app-state-repository.d.ts +25 -1
  25. package/dist/server/persistence/sqlite-state-repository.js +47 -4
  26. package/dist/server/runtime.d.ts +82 -1
  27. package/dist/server/runtime.js +128 -7
  28. package/dist/server/server-context.d.ts +3 -0
  29. package/dist/server/store.d.ts +153 -68
  30. package/dist/server/store.js +405 -187
  31. package/dist/server/tool-llm.d.ts +65 -0
  32. package/dist/server/tool-llm.js +139 -0
  33. package/dist/server/tool-registry.d.ts +65 -0
  34. package/dist/server/tool-registry.js +129 -6
  35. package/dist/server/tool-runner.d.ts +70 -0
  36. package/dist/server/tool-runner.js +210 -10
  37. package/dist/server/types.d.ts +97 -65
  38. package/package.json +166 -155
  39. package/dist/client/assets/index-8Pf3MG2d.js +0 -72
  40. package/dist/client/assets/index-Jnj15dhT.css +0 -1
@@ -0,0 +1,302 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { existsSync } from "node:fs";
3
+ import { mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
4
+ import { join, resolve } from "node:path";
5
+ /**
6
+ * The model, and the only one this feature can use.
7
+ *
8
+ * WHY THIS ONE. `bge-small-zh` scores 0.618 nDCG@10 averaged over the Chinese retrieval tasks it is
9
+ * benchmarked on - against 0.695 for `bge-base-zh`, which is four times the download. `gte-multilingual-base`
10
+ * scores higher on both Chinese and English, at 324 MB against this one's 23 MB. For a workspace-sized
11
+ * corpus the next thing that would help is not a bigger model but fusing this one with the lexical
12
+ * channels, which costs nothing and is what `KnowledgeBase.search` does.
13
+ */
14
+ export const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
15
+ /**
16
+ * HOW BIG ONE EMBEDDED PIECE MAY BE, in characters.
17
+ *
18
+ * THIS NUMBER IS A MODEL LIMIT, NOT A TUNING KNOB, and getting it wrong is invisible. The model
19
+ * declares `max_position_embeddings: 512`, and a Chinese character costs roughly one token, so a piece
20
+ * is a little under half the window: 300 characters leaves room for the `[CLS]`/`[SEP]` pair and for
21
+ * tokenisation that splits a few characters into two. The retrieval chunks are 1200 characters - more
22
+ * than double this - which is fine for something a MODEL reads and fatal for something a MODEL
23
+ * ENCODES: handed a 1200-character chunk, roughly 60% of it is silently discarded past the window.
24
+ * English would survive it, so the degradation would land only on Chinese: the language this project
25
+ * is written in.
26
+ */
27
+ export const EMBED_PIECE_CHARS = 300;
28
+ const EMBED_PIECE_OVERLAP = 40;
29
+ /** Per query. Cosine similarity of unit vectors; beyond this the passage is noise. */
30
+ const MIN_COSINE = 0.15;
31
+ /**
32
+ * Splits a retrieval chunk into pieces small enough for the model's window.
33
+ *
34
+ * WHY NOT SHRINK THE RETRIEVAL CHUNKS INSTEAD. 1200 characters is the right size for a passage a model
35
+ * READS - it holds enough context to answer from. Encoding is a different constraint from reading, and
36
+ * conflating them would trade retrieval quality for a convenience the encoder does not need.
37
+ */
38
+ export function embedPieces(text) {
39
+ const pieces = [];
40
+ for (let offset = 0; offset < text.length; offset += EMBED_PIECE_CHARS - EMBED_PIECE_OVERLAP) {
41
+ const piece = text.slice(offset, offset + EMBED_PIECE_CHARS).trim();
42
+ if (piece)
43
+ pieces.push(piece);
44
+ if (offset + EMBED_PIECE_CHARS >= text.length)
45
+ break;
46
+ }
47
+ return pieces.length ? pieces : [text.slice(0, EMBED_PIECE_CHARS)];
48
+ }
49
+ /** Element-wise max over the pieces, then unit length. See `pool` for why max and not mean. */
50
+ export function pool(vectors) {
51
+ if (!vectors.length)
52
+ return [];
53
+ const pooled = new Array(vectors[0].length).fill(Number.NEGATIVE_INFINITY);
54
+ for (const vector of vectors) {
55
+ for (let i = 0; i < vector.length; i++)
56
+ if (vector[i] > pooled[i])
57
+ pooled[i] = vector[i];
58
+ }
59
+ let norm = 0;
60
+ for (const value of pooled)
61
+ norm += value * value;
62
+ norm = Math.sqrt(norm) || 1;
63
+ return pooled.map(value => value / norm);
64
+ }
65
+ export function cosine(a, b) {
66
+ let dot = 0;
67
+ for (let i = 0; i < a.length && i < b.length; i++)
68
+ dot += a[i] * b[i];
69
+ return dot;
70
+ }
71
+ /**
72
+ * The optional vector half of retrieval.
73
+ *
74
+ * EVERY FAILURE HERE IS A FALLBACK TO LEXICAL, NEVER AN ERROR. The model is an optional npm package
75
+ * that a skipped optional install can take away, and it is a 512-token window
76
+ * wrapped around a Chinese corpus. Any of those failing is a reason to answer from the lexical
77
+ * channels and say so - never a reason to fail a search that the lexical channels can already serve.
78
+ */
79
+ export class VectorIndex {
80
+ directory;
81
+ extractor;
82
+ loading;
83
+ failure;
84
+ cache = new Map();
85
+ pending = new Set();
86
+ queue = [];
87
+ background;
88
+ constructor(directory) {
89
+ this.directory = directory;
90
+ }
91
+ /**
92
+ * Where the shipped weights live, relative to this module.
93
+ *
94
+ * `dist/server/knowledge-vectors.js` is two levels below the package root both in a published
95
+ * package and in this repository's own build - the same `../../assets/...` shape `runtime.ts` uses to
96
+ * reach `agents/`. The directory keeps the upstream id in its path because that is how the library
97
+ * looks the model up under `localModelPath`.
98
+ */
99
+ bundledModel() { return resolve(import.meta.dirname, "../../assets/knowledge-model"); }
100
+ enabled() { return process.env.SUPER_PI_KNOWLEDGE_EMBEDDINGS !== "0"; }
101
+ status(indexed) {
102
+ return {
103
+ enabled: this.enabled(), ready: Boolean(this.extractor), indexed, error: this.failure,
104
+ // Whether the shipped weights are on disk, reported before the model is ever loaded: "the
105
+ // package was built without its assets" and "the model failed to load" are different problems.
106
+ bundled: existsSync(this.bundledModel()),
107
+ };
108
+ }
109
+ workspace(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
110
+ folder(workspaceId) { return join(this.workspace(workspaceId), "vectors"); }
111
+ file(workspaceId, documentId) { return join(this.folder(workspaceId), `${documentId}.vec.json`); }
112
+ /**
113
+ * Loads the model once, from the copy that ships with this package. The `dtype` is not optional: on
114
+ * Node the library defaults to fp32, which for this model is 90 MB rather than 23.
115
+ *
116
+ * THERE IS NO NETWORK HERE, and that is the whole point of vendoring the weights. The first version
117
+ * downloaded the model on first use, which put a 23 MB fetch behind a host the machine may not be
118
+ * able to reach, on the first search, for every person who installed the package. Shipping the
119
+ * weights turns a failure mode into a file, and makes the feature work on a machine that has never
120
+ * been online since it was installed.
121
+ */
122
+ load() {
123
+ if (this.extractor)
124
+ return Promise.resolve(this.extractor);
125
+ if (this.failure)
126
+ return Promise.resolve(undefined);
127
+ this.loading ??= (async () => {
128
+ try {
129
+ if (!existsSync(this.bundledModel()))
130
+ throw new Error(`随包分发的模型缺失:${this.bundledModel()}`);
131
+ // THE SPECIFIER GOES THROUGH A VARIABLE, and that is deliberate. Written as a literal,
132
+ // `tsc` resolves it while compiling - so a machine whose optional install skipped this package
133
+ // could not build the server at all, which makes an "optional" dependency mandatory in the
134
+ // only sense that matters to a person who installed it. Node resolves the variable at RUN time
135
+ // instead, where its absence is already handled: the throw below is the same branch that
136
+ // handles a missing model.
137
+ const specifier = "@huggingface/transformers";
138
+ const { pipeline, env } = await import(specifier);
139
+ // The library resolves `<localModelPath>/<model id>/onnx/...`, which is why the vendored tree
140
+ // keeps the `Xenova/bge-small-zh-v1.5` directory rather than being flattened.
141
+ env.localModelPath = this.bundledModel();
142
+ env.allowRemoteModels = false;
143
+ this.extractor = await pipeline("feature-extraction", KNOWLEDGE_MODEL, { dtype: "q8" });
144
+ return this.extractor;
145
+ }
146
+ catch (error) {
147
+ this.failure = error instanceof Error ? error.message : String(error);
148
+ console.warn(`Super Pi: 语义检索不可用,将只用关键词检索:${this.failure}`);
149
+ return undefined;
150
+ }
151
+ finally {
152
+ this.loading = undefined;
153
+ }
154
+ })();
155
+ return this.loading;
156
+ }
157
+ async embed(texts) {
158
+ if (!texts.length || !this.enabled())
159
+ return null;
160
+ const extractor = await this.load();
161
+ if (!extractor)
162
+ return null;
163
+ try {
164
+ const output = await extractor(texts, { pooling: "cls", normalize: true });
165
+ const dimension = output.dims[output.dims.length - 1];
166
+ return Array.from({ length: output.dims[0] }, (_, row) => Array.from(output.data.slice(row * dimension, (row + 1) * dimension)));
167
+ }
168
+ catch (error) {
169
+ this.failure = error instanceof Error ? error.message : String(error);
170
+ console.warn(`Super Pi: 生成向量失败,将只用关键词检索:${this.failure}`);
171
+ return null;
172
+ }
173
+ }
174
+ /** Vectors already stored for one workspace, keyed by document then chunk index. */
175
+ async read(workspaceId, documentId) {
176
+ try {
177
+ const document = JSON.parse(await readFile(this.file(workspaceId, documentId), "utf8"));
178
+ return Array.isArray(document?.records) ? document : null;
179
+ }
180
+ catch (error) {
181
+ if (error.code === "ENOENT")
182
+ return null;
183
+ return null;
184
+ }
185
+ }
186
+ async count(workspaceId) {
187
+ try {
188
+ const names = (await readdir(this.folder(workspaceId))).filter(name => name.endsWith(".vec.json"));
189
+ let total = 0;
190
+ for (const name of names) {
191
+ const document = await this.read(workspaceId, name.slice(0, -".vec.json".length));
192
+ total += document?.records.length ?? 0;
193
+ }
194
+ return total;
195
+ }
196
+ catch {
197
+ return 0;
198
+ }
199
+ }
200
+ /** Drops a document's vectors. Called whenever its content changes or it is deleted. */
201
+ async forget(workspaceId, documentId) {
202
+ this.cache.delete(`${workspaceId}/${documentId}`);
203
+ await unlink(this.file(workspaceId, documentId)).catch(() => { });
204
+ }
205
+ /**
206
+ * Embeds anything not already stored, WITHOUT waiting for it.
207
+ *
208
+ * WHY IT IS NOT AWAITED. This runs after an import or on the first search of a session, and the
209
+ * first run of it downloads a model. Neither should make a person wait: the import returns as soon
210
+ * as the document is stored and the search returns lexical hits, and a later search is simply better
211
+ * because by then the vectors exist.
212
+ *
213
+ * WHY REQUESTS QUEUE RATHER THAN DROP. One job runs at a time - two would load the same model twice
214
+ * and race each other's writes - but a request arriving mid-job has to be remembered. Dropping it
215
+ * would mean a document imported while a background index was in flight stayed unindexed until the
216
+ * next restart, which is the kind of bug that only shows up on a busy machine weeks later.
217
+ */
218
+ schedule(workspaceId, documents) {
219
+ if (!this.enabled() || !documents.length)
220
+ return;
221
+ this.queue.push({ workspaceId, documents });
222
+ if (this.background)
223
+ return;
224
+ this.background = (async () => {
225
+ while (this.queue.length) {
226
+ const next = this.queue.shift();
227
+ try {
228
+ for (const document of next.documents) {
229
+ const key = `${next.workspaceId}/${document.id}`;
230
+ if (this.pending.has(key))
231
+ continue;
232
+ this.pending.add(key);
233
+ const stored = await this.read(next.workspaceId, document.id);
234
+ if (stored?.digest === document.digest && stored.records.length === document.chunks.length) {
235
+ this.pending.delete(key);
236
+ continue;
237
+ }
238
+ const vectors = [];
239
+ for (const chunk of document.chunks) {
240
+ const embedded = await this.embed(embedPieces(chunk.text));
241
+ if (!embedded) {
242
+ this.pending.delete(key);
243
+ return;
244
+ }
245
+ vectors.push({ chunk: chunk.index, values: pool(embedded) });
246
+ }
247
+ const folder = this.folder(next.workspaceId);
248
+ await mkdir(folder, { recursive: true });
249
+ const file = this.file(next.workspaceId, document.id), temporary = `${file}.${randomUUID()}.tmp`;
250
+ await writeFile(temporary, JSON.stringify({ digest: document.digest, records: vectors }), { encoding: "utf8", mode: 0o600 });
251
+ await rename(temporary, file);
252
+ this.cache.set(key, new Map(vectors.map(record => [record.chunk, record])));
253
+ this.pending.delete(key);
254
+ }
255
+ }
256
+ catch (error) {
257
+ console.warn(`Super Pi: 语义索引失败,将只用关键词检索:${error instanceof Error ? error.message : String(error)}`);
258
+ }
259
+ }
260
+ })().finally(() => { this.background = undefined; });
261
+ }
262
+ /**
263
+ * Cosine of every stored chunk against the query, best first.
264
+ *
265
+ * WHY THE RAW COSINE IS NOT RE-NORMALISED BY THE BEST HIT. Every lexical channel is divided by its
266
+ * own maximum, and that is right for them because BM25 scores are unbounded and have no absolute
267
+ * meaning. Cosine is the opposite: it is already a bounded, interpretable quantity, and dividing it
268
+ * by the best hit of the query would turn "nothing here is close" into "the best one is 1.0". The
269
+ * knowledge base's most useful property is that it can say it does not know, and rescaling is exactly
270
+ * how that gets spent.
271
+ */
272
+ async search(workspaceId, query, keys) {
273
+ if (!this.enabled() || !keys.length)
274
+ return null;
275
+ const documentIds = new Set(keys.map(([, documentId]) => documentId));
276
+ // First query of the session: read what was indexed last time rather than assuming nothing is.
277
+ for (const documentId of documentIds) {
278
+ if (this.cache.has(`${workspaceId}/${documentId}`))
279
+ continue;
280
+ const document = await this.read(workspaceId, documentId);
281
+ if (document)
282
+ this.cache.set(`${workspaceId}/${documentId}`, new Map(document.records.map(record => [record.chunk, record])));
283
+ }
284
+ const available = keys.filter(([, documentId]) => this.cache.get(`${workspaceId}/${documentId}`)?.size);
285
+ if (!available.length)
286
+ return null;
287
+ const vector = await this.embed([query]);
288
+ if (!vector?.[0])
289
+ return null;
290
+ const scores = new Map();
291
+ for (const [key, documentId] of available) {
292
+ const chunk = Number(key.split("#")[1]);
293
+ const record = this.cache.get(`${workspaceId}/${documentId}`)?.get(chunk);
294
+ if (!record)
295
+ continue;
296
+ const similarity = cosine(vector[0], record.values);
297
+ if (similarity >= MIN_COSINE)
298
+ scores.set(key, similarity);
299
+ }
300
+ return scores;
301
+ }
302
+ }
@@ -0,0 +1,263 @@
1
+ import { VectorIndex } from "./knowledge-vectors.js";
2
+ import type { Store } from "./store.js";
3
+ import type { Conversation } from "./types.js";
4
+ /** One document as it lives on disk: the ORIGINAL text, never the chunks. */
5
+ export type KnowledgeDocument = {
6
+ id: string;
7
+ workspaceId: string;
8
+ title: string;
9
+ /** Where it came from - a workspace-relative path, or `pasted` for hand-entered text. */
10
+ source: string;
11
+ format: string;
12
+ characters: number;
13
+ createdAt: string;
14
+ updatedAt: string;
15
+ /**
16
+ * Content fingerprint, used to recognise an import the user already made.
17
+ *
18
+ * WHY IT IS STORED RATHER THAN COMPUTED ON COMPARISON. Deduplication runs inside `add`, which already
19
+ * reads every document in the workspace; hashing each one's full text there would be O(corpus) on
20
+ * every import. Storing it turns the check into one string comparison, and a document written by an
21
+ * older build simply hashes on read (`digestOf`) instead.
22
+ */
23
+ digest: string;
24
+ text: string;
25
+ };
26
+ /** A retrievable slice. `heading` is the nearest preceding markdown heading, so a hit can be cited. */
27
+ export type KnowledgeChunk = {
28
+ index: number;
29
+ heading: string;
30
+ text: string;
31
+ };
32
+ export type KnowledgeHit = {
33
+ documentId: string;
34
+ title: string;
35
+ source: string;
36
+ heading: string;
37
+ chunk: number;
38
+ score: number;
39
+ text: string;
40
+ };
41
+ /** 1 MB. Big enough for a design doc or a spec, small enough that chunking stays instant. */
42
+ export declare const MAX_DOCUMENT_BYTES: number;
43
+ /** A pasted document rides inside a JSON body, so the request cap leaves room for escaping overhead. */
44
+ export declare const MAX_KNOWLEDGE_REQUEST_BYTES: number;
45
+ /** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
46
+ export declare const MAX_WORKSPACE_BYTES: number;
47
+ export declare const DEFAULT_RESULT_LIMIT = 6;
48
+ export declare const MAX_RESULT_LIMIT = 20;
49
+ /** What one search may be asked for. `rerank` and `conversation` exist only for the agent's own calls. */
50
+ export type KnowledgeSearch = {
51
+ limit?: number;
52
+ /** Re-order with the session's model. Off for the settings preview, which must stay instant. */
53
+ rerank?: boolean;
54
+ /** Required by `rerank`: it answers with this conversation's model and workspace. */
55
+ conversation?: Conversation;
56
+ signal?: AbortSignal;
57
+ };
58
+ /**
59
+ * The slice of the runtime the reranker needs, declared structurally.
60
+ *
61
+ * Declared here rather than imported so this file does not depend on the spawn machinery that depends
62
+ * on it - `runtime.ts` imports `KNOWLEDGE_TOOL_NAME` from here, and a type-only back edge into that
63
+ * module would make the cycle visible in every import graph.
64
+ */
65
+ export type KnowledgeRuntime = {
66
+ answer(input: {
67
+ conversation: Conversation;
68
+ prompt: string;
69
+ cwd: string;
70
+ signal: AbortSignal;
71
+ knowledgeRerank?: boolean;
72
+ }): AsyncIterable<string>;
73
+ };
74
+ /**
75
+ * The first integer array in the model's answer, or null.
76
+ *
77
+ * WHY TOLERANT PARSING. The prompt asks for a bare JSON array, and a model that mostly complies will
78
+ * still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
79
+ * guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
80
+ * the lexical order.
81
+ */
82
+ export declare function parseRanking(output: string): number[] | null;
83
+ /** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
84
+ export declare const KNOWLEDGE_TOOL_NAME = "knowledge_search";
85
+ /**
86
+ * Index terms for one string: CJK bigrams plus Latin words.
87
+ *
88
+ * WHY BIGRAMS AND NOT WORDS. Chinese has no spaces, so the word-list tokenizer the rest of this
89
+ * server uses (`ActiveMemory.search`) returns one enormous "word" for a whole sentence and matches
90
+ * nothing usefully. Character bigrams need no dictionary and no dependency, and they carry enough
91
+ * precision that 知识库 and 记忆库 share 识库 but not the whole run - which is what makes a two-term
92
+ * overlap meaningful rather than accidental.
93
+ */
94
+ export declare function tokenize(text: string): string[];
95
+ /**
96
+ * The longer-range channel: character TRIGRAMS, for the passage that repeats a phrase verbatim.
97
+ *
98
+ * WHY A SECOND GRANULARITY AND NOT JUST BIGRAMS. Bigrams answer "do these share any two-character
99
+ * run", which is high recall but low precision - two unrelated Chinese sentences collide constantly on
100
+ * a shared pair. Trigrams only fire on a real three-character overlap, so when the user's question
101
+ * quotes the document's own wording this channel separates it sharply from the rest of the corpus.
102
+ * It is scored on its own and fused with the bigram channel rather than merged into it, because the
103
+ * two answer different questions and a term that is strong for one is usually weak for the other.
104
+ */
105
+ export declare function tokenizePhrases(text: string): string[];
106
+ /** The identity of a document's CONTENT, independent of its name, its path and when it arrived. */
107
+ export declare function digestOf(text: string): string;
108
+ /**
109
+ * Splits a document into overlapping chunks, remembering the heading each one sits under.
110
+ *
111
+ * WHY OVERLAP. A passage that spans a boundary is the passage most likely to be the answer - the
112
+ * sentence that defines a term usually starts in one chunk and ends in the next - and a retrieval
113
+ * system that cut there would return a fragment ending mid-thought. The 150-character tail is
114
+ * repeated at the head of the next chunk to keep that sentence whole in at least one hit.
115
+ */
116
+ export declare function chunkDocument(text: string): KnowledgeChunk[];
117
+ export declare class KnowledgeBase {
118
+ readonly directory: string;
119
+ private store;
120
+ private runtime?;
121
+ private vectors?;
122
+ private capabilities;
123
+ private indexes;
124
+ private serial;
125
+ constructor(directory: string, store: Store, runtime?: KnowledgeRuntime | undefined, vectors?: VectorIndex | undefined);
126
+ /** `SUPER_PI_KNOWLEDGE_ENABLED=0` turns the whole feature off, the same way memory turns off. */
127
+ enabled(): boolean;
128
+ /** The opaque workspace identity on disk, encoded. Paths never come from the model or the request. */
129
+ private folder;
130
+ private file;
131
+ private requireWorkspace;
132
+ private locked;
133
+ /** Metadata only - the caller listing documents must never receive the full text of a workspace. */
134
+ private summarise;
135
+ private readAll;
136
+ private save;
137
+ private index;
138
+ /**
139
+ * Loads (or rebuilds) one workspace's index. Cache miss is the only thing that reads the disk.
140
+ *
141
+ * WHY REBUILT PER WORKSPACE AND NOT ONE GLOBAL INDEX. Isolation is a security property here, not a
142
+ * performance one: a hit may only ever come from the workspace the turn's capability names. A
143
+ * single shared index would need every search to filter by workspace after scoring, which is one
144
+ * forgotten filter away from a cross-tenant leak. Separate caches cannot express that mistake.
145
+ *
146
+ * WHY THERE IS NO ON-DISK INDEX. Measured, not assumed: rebuilding the index of a 16 MB workspace -
147
+ * the feature's own capacity cap - costs about 180 ms, once per process. A serialised index would be
148
+ * larger than the corpus it describes and JSON.parse of it is not reliably cheaper than the
149
+ * tokenisation, so it would trade a correctness-neutral optimisation for a second on-disk artefact
150
+ * that can drift from the documents.
151
+ */
152
+ private load;
153
+ private invalidate;
154
+ list(workspaceId: string): Promise<Array<Omit<KnowledgeDocument, "text">>>;
155
+ /**
156
+ * Whether this workspace has any document at all, which is what decides whether the turn mounts the
157
+ * extension.
158
+ *
159
+ * WHY A DIRECTORY LISTING AND NOT `list()`. This runs once per turn, before every spawn. `list()`
160
+ * would read and parse every document's full text to answer a yes/no question, so a workspace with
161
+ * a dozen real documents would pay for parsing all of them on every message - including the
162
+ * catalogue fetch the extension makes the moment it loads. `readdir` answers the same question for
163
+ * the price of one directory entry, and a name is all this needs.
164
+ */
165
+ hasDocuments(workspaceId: string): Promise<boolean>;
166
+ /**
167
+ * Adds a document, either from a workspace file or from pasted text.
168
+ *
169
+ * WHY THE FILE IS READ HERE AND NOT UPLOADED. A knowledge base is made of documents the person
170
+ * ALREADY has in their workspace - re-uploading a copy of the file would give the agent a second,
171
+ * newer version that diverges from the one the workspace serves. The path therefore goes through
172
+ * the same `resolveInsideRoot` guard the file editor uses, so symlink and `..` escapes are refused
173
+ * by construction instead of by a check someone has to remember.
174
+ */
175
+ add(workspaceId: string, body: {
176
+ path?: string;
177
+ title?: string;
178
+ text?: string;
179
+ }): Promise<Omit<KnowledgeDocument, "text">>;
180
+ remove(workspaceId: string, documentId: string): Promise<void>;
181
+ /**
182
+ * Scores this workspace's chunks on four independent lexical channels and fuses them.
183
+ *
184
+ * WHY LEXICAL AND NOT ONLY EMBEDDINGS. See docs/KNOWLEDGE_BASE.md: the vector channel is fused in
185
+ * here rather than replacing this one, because each is wrong in a way the other catches. BM25
186
+ * misses a paraphrase completely; a vector search will happily return the nearest thing at a
187
+ * confident-looking 0.58 when the answer is not in the knowledge base at all. Summing a normalised
188
+ * lexical score with a normalised cosine score keeps both signals - and keeps "no hit" reachable.
189
+ *
190
+ * WHY NORMALISE PER CHANNEL AND SUM, RATHER THAN RRF. Reciprocal rank fusion is the usual way to
191
+ * combine ranked lists, and it would be a regression here: it discards score magnitude, so a
192
+ * passage that matches decisively and one that matches barely would arrive at the same rank and
193
+ * finish within a couple of percent of each other. The gap between a clear winner and a near miss is
194
+ * exactly what BM25 was measuring. Each channel is therefore divided by its own best score for this
195
+ * query, which preserves the shape of that channel's confidence while making the weights comparable.
196
+ */
197
+ search(workspaceId: string, query: string, options?: KnowledgeSearch): Promise<KnowledgeHit[]>;
198
+ /**
199
+ * Re-orders the candidate pool with the session's own model.
200
+ *
201
+ * WHY THIS EXISTS AND WHAT IT IS WORTH. Lexical retrieval ranks on shared characters, so a passage
202
+ * that answers in different words sits below passages that merely repeat the question's wording -
203
+ * often below the fold, where the character budget has already cut it off. Reading the candidates
204
+ * is something the model can already do, so it is asked to pick.
205
+ *
206
+ * ITS CEILING, STATED PLAINLY. Reranking can only reorder what the first stage returned. A query
207
+ * with NO lexical overlap at all never reaches this function, which is why the vector channel and
208
+ * not this one is what closes the paraphrase gap - see docs/KNOWLEDGE_BASE.md.
209
+ *
210
+ * WHY IT FAILS OPEN. Every outcome that is not a clean parse returns the lexical order. A reranker
211
+ * that made things worse, hung, or answered with prose would degrade the feature invisibly, and a
212
+ * retrieval layer that is allowed to get worse by being clever is worse than one that is not.
213
+ */
214
+ private rerank;
215
+ /**
216
+ * Mints a read-only token for one turn. The agent can retrieve, never write: everything a person
217
+ * puts in a knowledge base is theirs to curate from the settings window.
218
+ */
219
+ issue(conversationId: string): string;
220
+ release(token: string): void;
221
+ private identity;
222
+ /** The agent-facing endpoint: a title catalogue and a search. Nothing that writes. */
223
+ tool(token: string, body: {
224
+ kind?: string;
225
+ query?: string;
226
+ limit?: number;
227
+ }, signal?: AbortSignal): Promise<{
228
+ documents: {
229
+ title: string;
230
+ source: string;
231
+ characters: number;
232
+ }[];
233
+ hits?: undefined;
234
+ reranked?: undefined;
235
+ } | {
236
+ documents?: undefined;
237
+ hits: KnowledgeHit[];
238
+ reranked: boolean;
239
+ }>;
240
+ /** The browser-facing endpoint, behind the ordinary access token and same-origin check. */
241
+ manage(workspaceId: string, body: {
242
+ action?: string;
243
+ path?: string;
244
+ title?: string;
245
+ text?: string;
246
+ documentId?: string;
247
+ query?: string;
248
+ limit?: number;
249
+ }): Promise<{
250
+ hits: KnowledgeHit[];
251
+ documents: Omit<KnowledgeDocument, "text">[];
252
+ semantic: import("./knowledge-vectors.js").VectorStatus | null;
253
+ }>;
254
+ /**
255
+ * What the panel shows, including whether semantic retrieval is actually working.
256
+ *
257
+ * WHY THE STATE IS WORTH SHOWING. The vector half depends on an optional package and a download from
258
+ * a host the machine may not reach, and both fail quietly by design. Without this on screen, "the
259
+ * feature is off" is indistinguishable from "the feature is broken here" - which is the difference
260
+ * between not wanting a feature and not being able to have it.
261
+ */
262
+ private overview;
263
+ }