@evandu/super-pi 0.1.84 → 0.1.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/super-pi-knowledge/index.ts +40 -0
- package/agents/super-pi-task/index.ts +2 -2
- package/assets/knowledge-model/README.md +37 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/config.json +31 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/special_tokens_map.json +7 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/tokenizer.json +21278 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/tokenizer_config.json +15 -0
- package/assets/knowledge-model/Xenova/bge-small-zh-v1.5/vocab.txt +21128 -0
- package/dist/.super-pi-check-inputs.json +4 -0
- package/dist/client/.super-pi-client-sources.json +10 -7
- package/dist/client/assets/index-B3QVoYJY.css +1 -0
- package/dist/client/assets/index-C_giUkvT.js +72 -0
- package/dist/client/index.html +2 -2
- package/dist/server/application/run-service.d.ts +50 -11
- package/dist/server/application/run-service.js +206 -77
- package/dist/server/command-handler.js +16 -4
- package/dist/server/http-server.js +218 -8
- package/dist/server/index.js +15 -1
- package/dist/server/knowledge-vectors.d.ts +133 -0
- package/dist/server/knowledge-vectors.js +302 -0
- package/dist/server/knowledge.d.ts +263 -0
- package/dist/server/knowledge.js +743 -0
- package/dist/server/persistence/app-state-repository.d.ts +25 -1
- package/dist/server/persistence/sqlite-state-repository.js +47 -4
- package/dist/server/runtime.d.ts +82 -1
- package/dist/server/runtime.js +128 -7
- package/dist/server/server-context.d.ts +3 -0
- package/dist/server/store.d.ts +153 -68
- package/dist/server/store.js +405 -187
- package/dist/server/tool-llm.d.ts +65 -0
- package/dist/server/tool-llm.js +139 -0
- package/dist/server/tool-registry.d.ts +65 -0
- package/dist/server/tool-registry.js +129 -6
- package/dist/server/tool-runner.d.ts +70 -0
- package/dist/server/tool-runner.js +210 -10
- package/dist/server/types.d.ts +97 -65
- package/package.json +166 -155
- package/dist/client/assets/index-8Pf3MG2d.js +0 -72
- package/dist/client/assets/index-Jnj15dhT.css +0 -1
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
/**
|
|
6
|
+
* The model, and the only one this feature can use.
|
|
7
|
+
*
|
|
8
|
+
* WHY THIS ONE. `bge-small-zh` scores 0.618 nDCG@10 averaged over the Chinese retrieval tasks it is
|
|
9
|
+
* benchmarked on - against 0.695 for `bge-base-zh`, which is four times the download. `gte-multilingual-base`
|
|
10
|
+
* scores higher on both Chinese and English, at 324 MB against this one's 23 MB. For a workspace-sized
|
|
11
|
+
* corpus the next thing that would help is not a bigger model but fusing this one with the lexical
|
|
12
|
+
* channels, which costs nothing and is what `KnowledgeBase.search` does.
|
|
13
|
+
*/
|
|
14
|
+
export const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
|
|
15
|
+
/**
|
|
16
|
+
* HOW BIG ONE EMBEDDED PIECE MAY BE, in characters.
|
|
17
|
+
*
|
|
18
|
+
* THIS NUMBER IS A MODEL LIMIT, NOT A TUNING KNOB, and getting it wrong is invisible. The model
|
|
19
|
+
* declares `max_position_embeddings: 512`, and a Chinese character costs roughly one token, so a piece
|
|
20
|
+
* is a little under half the window: 300 characters leaves room for the `[CLS]`/`[SEP]` pair and for
|
|
21
|
+
* tokenisation that splits a few characters into two. The retrieval chunks are 1200 characters - more
|
|
22
|
+
* than double this - which is fine for something a MODEL reads and fatal for something a MODEL
|
|
23
|
+
* ENCODES: handed a 1200-character chunk, roughly 60% of it is silently discarded past the window.
|
|
24
|
+
* English would survive it, so the degradation would land only on Chinese: the language this project
|
|
25
|
+
* is written in.
|
|
26
|
+
*/
|
|
27
|
+
export const EMBED_PIECE_CHARS = 300;
|
|
28
|
+
const EMBED_PIECE_OVERLAP = 40;
|
|
29
|
+
/** Per query. Cosine similarity of unit vectors; beyond this the passage is noise. */
|
|
30
|
+
const MIN_COSINE = 0.15;
|
|
31
|
+
/**
|
|
32
|
+
* Splits a retrieval chunk into pieces small enough for the model's window.
|
|
33
|
+
*
|
|
34
|
+
* WHY NOT SHRINK THE RETRIEVAL CHUNKS INSTEAD. 1200 characters is the right size for a passage a model
|
|
35
|
+
* READS - it holds enough context to answer from. Encoding is a different constraint from reading, and
|
|
36
|
+
* conflating them would trade retrieval quality for a convenience the encoder does not need.
|
|
37
|
+
*/
|
|
38
|
+
export function embedPieces(text) {
|
|
39
|
+
const pieces = [];
|
|
40
|
+
for (let offset = 0; offset < text.length; offset += EMBED_PIECE_CHARS - EMBED_PIECE_OVERLAP) {
|
|
41
|
+
const piece = text.slice(offset, offset + EMBED_PIECE_CHARS).trim();
|
|
42
|
+
if (piece)
|
|
43
|
+
pieces.push(piece);
|
|
44
|
+
if (offset + EMBED_PIECE_CHARS >= text.length)
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
return pieces.length ? pieces : [text.slice(0, EMBED_PIECE_CHARS)];
|
|
48
|
+
}
|
|
49
|
+
/** Element-wise max over the pieces, then unit length. See `pool` for why max and not mean. */
|
|
50
|
+
export function pool(vectors) {
|
|
51
|
+
if (!vectors.length)
|
|
52
|
+
return [];
|
|
53
|
+
const pooled = new Array(vectors[0].length).fill(Number.NEGATIVE_INFINITY);
|
|
54
|
+
for (const vector of vectors) {
|
|
55
|
+
for (let i = 0; i < vector.length; i++)
|
|
56
|
+
if (vector[i] > pooled[i])
|
|
57
|
+
pooled[i] = vector[i];
|
|
58
|
+
}
|
|
59
|
+
let norm = 0;
|
|
60
|
+
for (const value of pooled)
|
|
61
|
+
norm += value * value;
|
|
62
|
+
norm = Math.sqrt(norm) || 1;
|
|
63
|
+
return pooled.map(value => value / norm);
|
|
64
|
+
}
|
|
65
|
+
export function cosine(a, b) {
|
|
66
|
+
let dot = 0;
|
|
67
|
+
for (let i = 0; i < a.length && i < b.length; i++)
|
|
68
|
+
dot += a[i] * b[i];
|
|
69
|
+
return dot;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* The optional vector half of retrieval.
|
|
73
|
+
*
|
|
74
|
+
* EVERY FAILURE HERE IS A FALLBACK TO LEXICAL, NEVER AN ERROR. The model is an optional npm package
|
|
75
|
+
* that a skipped optional install can take away, and it is a 512-token window
|
|
76
|
+
* wrapped around a Chinese corpus. Any of those failing is a reason to answer from the lexical
|
|
77
|
+
* channels and say so - never a reason to fail a search that the lexical channels can already serve.
|
|
78
|
+
*/
|
|
79
|
+
export class VectorIndex {
|
|
80
|
+
directory;
|
|
81
|
+
extractor;
|
|
82
|
+
loading;
|
|
83
|
+
failure;
|
|
84
|
+
cache = new Map();
|
|
85
|
+
pending = new Set();
|
|
86
|
+
queue = [];
|
|
87
|
+
background;
|
|
88
|
+
constructor(directory) {
|
|
89
|
+
this.directory = directory;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Where the shipped weights live, relative to this module.
|
|
93
|
+
*
|
|
94
|
+
* `dist/server/knowledge-vectors.js` is two levels below the package root both in a published
|
|
95
|
+
* package and in this repository's own build - the same `../../assets/...` shape `runtime.ts` uses to
|
|
96
|
+
* reach `agents/`. The directory keeps the upstream id in its path because that is how the library
|
|
97
|
+
* looks the model up under `localModelPath`.
|
|
98
|
+
*/
|
|
99
|
+
bundledModel() { return resolve(import.meta.dirname, "../../assets/knowledge-model"); }
|
|
100
|
+
enabled() { return process.env.SUPER_PI_KNOWLEDGE_EMBEDDINGS !== "0"; }
|
|
101
|
+
status(indexed) {
|
|
102
|
+
return {
|
|
103
|
+
enabled: this.enabled(), ready: Boolean(this.extractor), indexed, error: this.failure,
|
|
104
|
+
// Whether the shipped weights are on disk, reported before the model is ever loaded: "the
|
|
105
|
+
// package was built without its assets" and "the model failed to load" are different problems.
|
|
106
|
+
bundled: existsSync(this.bundledModel()),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
workspace(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
|
|
110
|
+
folder(workspaceId) { return join(this.workspace(workspaceId), "vectors"); }
|
|
111
|
+
file(workspaceId, documentId) { return join(this.folder(workspaceId), `${documentId}.vec.json`); }
|
|
112
|
+
/**
|
|
113
|
+
* Loads the model once, from the copy that ships with this package. The `dtype` is not optional: on
|
|
114
|
+
* Node the library defaults to fp32, which for this model is 90 MB rather than 23.
|
|
115
|
+
*
|
|
116
|
+
* THERE IS NO NETWORK HERE, and that is the whole point of vendoring the weights. The first version
|
|
117
|
+
* downloaded the model on first use, which put a 23 MB fetch behind a host the machine may not be
|
|
118
|
+
* able to reach, on the first search, for every person who installed the package. Shipping the
|
|
119
|
+
* weights turns a failure mode into a file, and makes the feature work on a machine that has never
|
|
120
|
+
* been online since it was installed.
|
|
121
|
+
*/
|
|
122
|
+
load() {
|
|
123
|
+
if (this.extractor)
|
|
124
|
+
return Promise.resolve(this.extractor);
|
|
125
|
+
if (this.failure)
|
|
126
|
+
return Promise.resolve(undefined);
|
|
127
|
+
this.loading ??= (async () => {
|
|
128
|
+
try {
|
|
129
|
+
if (!existsSync(this.bundledModel()))
|
|
130
|
+
throw new Error(`随包分发的模型缺失:${this.bundledModel()}`);
|
|
131
|
+
// THE SPECIFIER GOES THROUGH A VARIABLE, and that is deliberate. Written as a literal,
|
|
132
|
+
// `tsc` resolves it while compiling - so a machine whose optional install skipped this package
|
|
133
|
+
// could not build the server at all, which makes an "optional" dependency mandatory in the
|
|
134
|
+
// only sense that matters to a person who installed it. Node resolves the variable at RUN time
|
|
135
|
+
// instead, where its absence is already handled: the throw below is the same branch that
|
|
136
|
+
// handles a missing model.
|
|
137
|
+
const specifier = "@huggingface/transformers";
|
|
138
|
+
const { pipeline, env } = await import(specifier);
|
|
139
|
+
// The library resolves `<localModelPath>/<model id>/onnx/...`, which is why the vendored tree
|
|
140
|
+
// keeps the `Xenova/bge-small-zh-v1.5` directory rather than being flattened.
|
|
141
|
+
env.localModelPath = this.bundledModel();
|
|
142
|
+
env.allowRemoteModels = false;
|
|
143
|
+
this.extractor = await pipeline("feature-extraction", KNOWLEDGE_MODEL, { dtype: "q8" });
|
|
144
|
+
return this.extractor;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
this.failure = error instanceof Error ? error.message : String(error);
|
|
148
|
+
console.warn(`Super Pi: 语义检索不可用,将只用关键词检索:${this.failure}`);
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
finally {
|
|
152
|
+
this.loading = undefined;
|
|
153
|
+
}
|
|
154
|
+
})();
|
|
155
|
+
return this.loading;
|
|
156
|
+
}
|
|
157
|
+
async embed(texts) {
|
|
158
|
+
if (!texts.length || !this.enabled())
|
|
159
|
+
return null;
|
|
160
|
+
const extractor = await this.load();
|
|
161
|
+
if (!extractor)
|
|
162
|
+
return null;
|
|
163
|
+
try {
|
|
164
|
+
const output = await extractor(texts, { pooling: "cls", normalize: true });
|
|
165
|
+
const dimension = output.dims[output.dims.length - 1];
|
|
166
|
+
return Array.from({ length: output.dims[0] }, (_, row) => Array.from(output.data.slice(row * dimension, (row + 1) * dimension)));
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
this.failure = error instanceof Error ? error.message : String(error);
|
|
170
|
+
console.warn(`Super Pi: 生成向量失败,将只用关键词检索:${this.failure}`);
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/** Vectors already stored for one workspace, keyed by document then chunk index. */
|
|
175
|
+
async read(workspaceId, documentId) {
|
|
176
|
+
try {
|
|
177
|
+
const document = JSON.parse(await readFile(this.file(workspaceId, documentId), "utf8"));
|
|
178
|
+
return Array.isArray(document?.records) ? document : null;
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
if (error.code === "ENOENT")
|
|
182
|
+
return null;
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
async count(workspaceId) {
|
|
187
|
+
try {
|
|
188
|
+
const names = (await readdir(this.folder(workspaceId))).filter(name => name.endsWith(".vec.json"));
|
|
189
|
+
let total = 0;
|
|
190
|
+
for (const name of names) {
|
|
191
|
+
const document = await this.read(workspaceId, name.slice(0, -".vec.json".length));
|
|
192
|
+
total += document?.records.length ?? 0;
|
|
193
|
+
}
|
|
194
|
+
return total;
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return 0;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/** Drops a document's vectors. Called whenever its content changes or it is deleted. */
|
|
201
|
+
async forget(workspaceId, documentId) {
|
|
202
|
+
this.cache.delete(`${workspaceId}/${documentId}`);
|
|
203
|
+
await unlink(this.file(workspaceId, documentId)).catch(() => { });
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Embeds anything not already stored, WITHOUT waiting for it.
|
|
207
|
+
*
|
|
208
|
+
* WHY IT IS NOT AWAITED. This runs after an import or on the first search of a session, and the
|
|
209
|
+
* first run of it downloads a model. Neither should make a person wait: the import returns as soon
|
|
210
|
+
* as the document is stored and the search returns lexical hits, and a later search is simply better
|
|
211
|
+
* because by then the vectors exist.
|
|
212
|
+
*
|
|
213
|
+
* WHY REQUESTS QUEUE RATHER THAN DROP. One job runs at a time - two would load the same model twice
|
|
214
|
+
* and race each other's writes - but a request arriving mid-job has to be remembered. Dropping it
|
|
215
|
+
* would mean a document imported while a background index was in flight stayed unindexed until the
|
|
216
|
+
* next restart, which is the kind of bug that only shows up on a busy machine weeks later.
|
|
217
|
+
*/
|
|
218
|
+
schedule(workspaceId, documents) {
|
|
219
|
+
if (!this.enabled() || !documents.length)
|
|
220
|
+
return;
|
|
221
|
+
this.queue.push({ workspaceId, documents });
|
|
222
|
+
if (this.background)
|
|
223
|
+
return;
|
|
224
|
+
this.background = (async () => {
|
|
225
|
+
while (this.queue.length) {
|
|
226
|
+
const next = this.queue.shift();
|
|
227
|
+
try {
|
|
228
|
+
for (const document of next.documents) {
|
|
229
|
+
const key = `${next.workspaceId}/${document.id}`;
|
|
230
|
+
if (this.pending.has(key))
|
|
231
|
+
continue;
|
|
232
|
+
this.pending.add(key);
|
|
233
|
+
const stored = await this.read(next.workspaceId, document.id);
|
|
234
|
+
if (stored?.digest === document.digest && stored.records.length === document.chunks.length) {
|
|
235
|
+
this.pending.delete(key);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
const vectors = [];
|
|
239
|
+
for (const chunk of document.chunks) {
|
|
240
|
+
const embedded = await this.embed(embedPieces(chunk.text));
|
|
241
|
+
if (!embedded) {
|
|
242
|
+
this.pending.delete(key);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
vectors.push({ chunk: chunk.index, values: pool(embedded) });
|
|
246
|
+
}
|
|
247
|
+
const folder = this.folder(next.workspaceId);
|
|
248
|
+
await mkdir(folder, { recursive: true });
|
|
249
|
+
const file = this.file(next.workspaceId, document.id), temporary = `${file}.${randomUUID()}.tmp`;
|
|
250
|
+
await writeFile(temporary, JSON.stringify({ digest: document.digest, records: vectors }), { encoding: "utf8", mode: 0o600 });
|
|
251
|
+
await rename(temporary, file);
|
|
252
|
+
this.cache.set(key, new Map(vectors.map(record => [record.chunk, record])));
|
|
253
|
+
this.pending.delete(key);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
console.warn(`Super Pi: 语义索引失败,将只用关键词检索:${error instanceof Error ? error.message : String(error)}`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
})().finally(() => { this.background = undefined; });
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Cosine of every stored chunk against the query, best first.
|
|
264
|
+
*
|
|
265
|
+
* WHY THE RAW COSINE IS NOT RE-NORMALISED BY THE BEST HIT. Every lexical channel is divided by its
|
|
266
|
+
* own maximum, and that is right for them because BM25 scores are unbounded and have no absolute
|
|
267
|
+
* meaning. Cosine is the opposite: it is already a bounded, interpretable quantity, and dividing it
|
|
268
|
+
* by the best hit of the query would turn "nothing here is close" into "the best one is 1.0". The
|
|
269
|
+
* knowledge base's most useful property is that it can say it does not know, and rescaling is exactly
|
|
270
|
+
* how that gets spent.
|
|
271
|
+
*/
|
|
272
|
+
async search(workspaceId, query, keys) {
|
|
273
|
+
if (!this.enabled() || !keys.length)
|
|
274
|
+
return null;
|
|
275
|
+
const documentIds = new Set(keys.map(([, documentId]) => documentId));
|
|
276
|
+
// First query of the session: read what was indexed last time rather than assuming nothing is.
|
|
277
|
+
for (const documentId of documentIds) {
|
|
278
|
+
if (this.cache.has(`${workspaceId}/${documentId}`))
|
|
279
|
+
continue;
|
|
280
|
+
const document = await this.read(workspaceId, documentId);
|
|
281
|
+
if (document)
|
|
282
|
+
this.cache.set(`${workspaceId}/${documentId}`, new Map(document.records.map(record => [record.chunk, record])));
|
|
283
|
+
}
|
|
284
|
+
const available = keys.filter(([, documentId]) => this.cache.get(`${workspaceId}/${documentId}`)?.size);
|
|
285
|
+
if (!available.length)
|
|
286
|
+
return null;
|
|
287
|
+
const vector = await this.embed([query]);
|
|
288
|
+
if (!vector?.[0])
|
|
289
|
+
return null;
|
|
290
|
+
const scores = new Map();
|
|
291
|
+
for (const [key, documentId] of available) {
|
|
292
|
+
const chunk = Number(key.split("#")[1]);
|
|
293
|
+
const record = this.cache.get(`${workspaceId}/${documentId}`)?.get(chunk);
|
|
294
|
+
if (!record)
|
|
295
|
+
continue;
|
|
296
|
+
const similarity = cosine(vector[0], record.values);
|
|
297
|
+
if (similarity >= MIN_COSINE)
|
|
298
|
+
scores.set(key, similarity);
|
|
299
|
+
}
|
|
300
|
+
return scores;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { VectorIndex } from "./knowledge-vectors.js";
|
|
2
|
+
import type { Store } from "./store.js";
|
|
3
|
+
import type { Conversation } from "./types.js";
|
|
4
|
+
/** One document as it lives on disk: the ORIGINAL text, never the chunks. */
|
|
5
|
+
export type KnowledgeDocument = {
|
|
6
|
+
id: string;
|
|
7
|
+
workspaceId: string;
|
|
8
|
+
title: string;
|
|
9
|
+
/** Where it came from - a workspace-relative path, or `pasted` for hand-entered text. */
|
|
10
|
+
source: string;
|
|
11
|
+
format: string;
|
|
12
|
+
characters: number;
|
|
13
|
+
createdAt: string;
|
|
14
|
+
updatedAt: string;
|
|
15
|
+
/**
|
|
16
|
+
* Content fingerprint, used to recognise an import the user already made.
|
|
17
|
+
*
|
|
18
|
+
* WHY IT IS STORED RATHER THAN COMPUTED ON COMPARISON. Deduplication runs inside `add`, which already
|
|
19
|
+
* reads every document in the workspace; hashing each one's full text there would be O(corpus) on
|
|
20
|
+
* every import. Storing it turns the check into one string comparison, and a document written by an
|
|
21
|
+
* older build simply hashes on read (`digestOf`) instead.
|
|
22
|
+
*/
|
|
23
|
+
digest: string;
|
|
24
|
+
text: string;
|
|
25
|
+
};
|
|
26
|
+
/** A retrievable slice. `heading` is the nearest preceding markdown heading, so a hit can be cited. */
|
|
27
|
+
export type KnowledgeChunk = {
|
|
28
|
+
index: number;
|
|
29
|
+
heading: string;
|
|
30
|
+
text: string;
|
|
31
|
+
};
|
|
32
|
+
export type KnowledgeHit = {
|
|
33
|
+
documentId: string;
|
|
34
|
+
title: string;
|
|
35
|
+
source: string;
|
|
36
|
+
heading: string;
|
|
37
|
+
chunk: number;
|
|
38
|
+
score: number;
|
|
39
|
+
text: string;
|
|
40
|
+
};
|
|
41
|
+
/** 1 MB. Big enough for a design doc or a spec, small enough that chunking stays instant. */
|
|
42
|
+
export declare const MAX_DOCUMENT_BYTES: number;
|
|
43
|
+
/** A pasted document rides inside a JSON body, so the request cap leaves room for escaping overhead. */
|
|
44
|
+
export declare const MAX_KNOWLEDGE_REQUEST_BYTES: number;
|
|
45
|
+
/** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
|
|
46
|
+
export declare const MAX_WORKSPACE_BYTES: number;
|
|
47
|
+
export declare const DEFAULT_RESULT_LIMIT = 6;
|
|
48
|
+
export declare const MAX_RESULT_LIMIT = 20;
|
|
49
|
+
/** What one search may be asked for. `rerank` and `conversation` exist only for the agent's own calls. */
|
|
50
|
+
export type KnowledgeSearch = {
|
|
51
|
+
limit?: number;
|
|
52
|
+
/** Re-order with the session's model. Off for the settings preview, which must stay instant. */
|
|
53
|
+
rerank?: boolean;
|
|
54
|
+
/** Required by `rerank`: it answers with this conversation's model and workspace. */
|
|
55
|
+
conversation?: Conversation;
|
|
56
|
+
signal?: AbortSignal;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* The slice of the runtime the reranker needs, declared structurally.
|
|
60
|
+
*
|
|
61
|
+
* Declared here rather than imported so this file does not depend on the spawn machinery that depends
|
|
62
|
+
* on it - `runtime.ts` imports `KNOWLEDGE_TOOL_NAME` from here, and a type-only back edge into that
|
|
63
|
+
* module would make the cycle visible in every import graph.
|
|
64
|
+
*/
|
|
65
|
+
export type KnowledgeRuntime = {
|
|
66
|
+
answer(input: {
|
|
67
|
+
conversation: Conversation;
|
|
68
|
+
prompt: string;
|
|
69
|
+
cwd: string;
|
|
70
|
+
signal: AbortSignal;
|
|
71
|
+
knowledgeRerank?: boolean;
|
|
72
|
+
}): AsyncIterable<string>;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* The first integer array in the model's answer, or null.
|
|
76
|
+
*
|
|
77
|
+
* WHY TOLERANT PARSING. The prompt asks for a bare JSON array, and a model that mostly complies will
|
|
78
|
+
* still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
|
|
79
|
+
* guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
|
|
80
|
+
* the lexical order.
|
|
81
|
+
*/
|
|
82
|
+
export declare function parseRanking(output: string): number[] | null;
|
|
83
|
+
/** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
|
|
84
|
+
export declare const KNOWLEDGE_TOOL_NAME = "knowledge_search";
|
|
85
|
+
/**
|
|
86
|
+
* Index terms for one string: CJK bigrams plus Latin words.
|
|
87
|
+
*
|
|
88
|
+
* WHY BIGRAMS AND NOT WORDS. Chinese has no spaces, so the word-list tokenizer the rest of this
|
|
89
|
+
* server uses (`ActiveMemory.search`) returns one enormous "word" for a whole sentence and matches
|
|
90
|
+
* nothing usefully. Character bigrams need no dictionary and no dependency, and they carry enough
|
|
91
|
+
* precision that 知识库 and 记忆库 share 识库 but not the whole run - which is what makes a two-term
|
|
92
|
+
* overlap meaningful rather than accidental.
|
|
93
|
+
*/
|
|
94
|
+
export declare function tokenize(text: string): string[];
|
|
95
|
+
/**
|
|
96
|
+
* The longer-range channel: character TRIGRAMS, for the passage that repeats a phrase verbatim.
|
|
97
|
+
*
|
|
98
|
+
* WHY A SECOND GRANULARITY AND NOT JUST BIGRAMS. Bigrams answer "do these share any two-character
|
|
99
|
+
* run", which is high recall but low precision - two unrelated Chinese sentences collide constantly on
|
|
100
|
+
* a shared pair. Trigrams only fire on a real three-character overlap, so when the user's question
|
|
101
|
+
* quotes the document's own wording this channel separates it sharply from the rest of the corpus.
|
|
102
|
+
* It is scored on its own and fused with the bigram channel rather than merged into it, because the
|
|
103
|
+
* two answer different questions and a term that is strong for one is usually weak for the other.
|
|
104
|
+
*/
|
|
105
|
+
export declare function tokenizePhrases(text: string): string[];
|
|
106
|
+
/** The identity of a document's CONTENT, independent of its name, its path and when it arrived. */
|
|
107
|
+
export declare function digestOf(text: string): string;
|
|
108
|
+
/**
|
|
109
|
+
* Splits a document into overlapping chunks, remembering the heading each one sits under.
|
|
110
|
+
*
|
|
111
|
+
* WHY OVERLAP. A passage that spans a boundary is the passage most likely to be the answer - the
|
|
112
|
+
* sentence that defines a term usually starts in one chunk and ends in the next - and a retrieval
|
|
113
|
+
* system that cut there would return a fragment ending mid-thought. The 150-character tail is
|
|
114
|
+
* repeated at the head of the next chunk to keep that sentence whole in at least one hit.
|
|
115
|
+
*/
|
|
116
|
+
export declare function chunkDocument(text: string): KnowledgeChunk[];
|
|
117
|
+
export declare class KnowledgeBase {
|
|
118
|
+
readonly directory: string;
|
|
119
|
+
private store;
|
|
120
|
+
private runtime?;
|
|
121
|
+
private vectors?;
|
|
122
|
+
private capabilities;
|
|
123
|
+
private indexes;
|
|
124
|
+
private serial;
|
|
125
|
+
constructor(directory: string, store: Store, runtime?: KnowledgeRuntime | undefined, vectors?: VectorIndex | undefined);
|
|
126
|
+
/** `SUPER_PI_KNOWLEDGE_ENABLED=0` turns the whole feature off, the same way memory turns off. */
|
|
127
|
+
enabled(): boolean;
|
|
128
|
+
/** The opaque workspace identity on disk, encoded. Paths never come from the model or the request. */
|
|
129
|
+
private folder;
|
|
130
|
+
private file;
|
|
131
|
+
private requireWorkspace;
|
|
132
|
+
private locked;
|
|
133
|
+
/** Metadata only - the caller listing documents must never receive the full text of a workspace. */
|
|
134
|
+
private summarise;
|
|
135
|
+
private readAll;
|
|
136
|
+
private save;
|
|
137
|
+
private index;
|
|
138
|
+
/**
|
|
139
|
+
* Loads (or rebuilds) one workspace's index. Cache miss is the only thing that reads the disk.
|
|
140
|
+
*
|
|
141
|
+
* WHY REBUILT PER WORKSPACE AND NOT ONE GLOBAL INDEX. Isolation is a security property here, not a
|
|
142
|
+
* performance one: a hit may only ever come from the workspace the turn's capability names. A
|
|
143
|
+
* single shared index would need every search to filter by workspace after scoring, which is one
|
|
144
|
+
* forgotten filter away from a cross-tenant leak. Separate caches cannot express that mistake.
|
|
145
|
+
*
|
|
146
|
+
* WHY THERE IS NO ON-DISK INDEX. Measured, not assumed: rebuilding the index of a 16 MB workspace -
|
|
147
|
+
* the feature's own capacity cap - costs about 180 ms, once per process. A serialised index would be
|
|
148
|
+
* larger than the corpus it describes and JSON.parse of it is not reliably cheaper than the
|
|
149
|
+
* tokenisation, so it would trade a correctness-neutral optimisation for a second on-disk artefact
|
|
150
|
+
* that can drift from the documents.
|
|
151
|
+
*/
|
|
152
|
+
private load;
|
|
153
|
+
private invalidate;
|
|
154
|
+
list(workspaceId: string): Promise<Array<Omit<KnowledgeDocument, "text">>>;
|
|
155
|
+
/**
|
|
156
|
+
* Whether this workspace has any document at all, which is what decides whether the turn mounts the
|
|
157
|
+
* extension.
|
|
158
|
+
*
|
|
159
|
+
* WHY A DIRECTORY LISTING AND NOT `list()`. This runs once per turn, before every spawn. `list()`
|
|
160
|
+
* would read and parse every document's full text to answer a yes/no question, so a workspace with
|
|
161
|
+
* a dozen real documents would pay for parsing all of them on every message - including the
|
|
162
|
+
* catalogue fetch the extension makes the moment it loads. `readdir` answers the same question for
|
|
163
|
+
* the price of one directory entry, and a name is all this needs.
|
|
164
|
+
*/
|
|
165
|
+
hasDocuments(workspaceId: string): Promise<boolean>;
|
|
166
|
+
/**
|
|
167
|
+
* Adds a document, either from a workspace file or from pasted text.
|
|
168
|
+
*
|
|
169
|
+
* WHY THE FILE IS READ HERE AND NOT UPLOADED. A knowledge base is made of documents the person
|
|
170
|
+
* ALREADY has in their workspace - re-uploading a copy of the file would give the agent a second,
|
|
171
|
+
* newer version that diverges from the one the workspace serves. The path therefore goes through
|
|
172
|
+
* the same `resolveInsideRoot` guard the file editor uses, so symlink and `..` escapes are refused
|
|
173
|
+
* by construction instead of by a check someone has to remember.
|
|
174
|
+
*/
|
|
175
|
+
add(workspaceId: string, body: {
|
|
176
|
+
path?: string;
|
|
177
|
+
title?: string;
|
|
178
|
+
text?: string;
|
|
179
|
+
}): Promise<Omit<KnowledgeDocument, "text">>;
|
|
180
|
+
remove(workspaceId: string, documentId: string): Promise<void>;
|
|
181
|
+
/**
|
|
182
|
+
* Scores this workspace's chunks on four independent lexical channels and fuses them.
|
|
183
|
+
*
|
|
184
|
+
* WHY LEXICAL AND NOT ONLY EMBEDDINGS. See docs/KNOWLEDGE_BASE.md: the vector channel is fused in
|
|
185
|
+
* here rather than replacing this one, because each is wrong in a way the other catches. BM25
|
|
186
|
+
* misses a paraphrase completely; a vector search will happily return the nearest thing at a
|
|
187
|
+
* confident-looking 0.58 when the answer is not in the knowledge base at all. Summing a normalised
|
|
188
|
+
* lexical score with a normalised cosine score keeps both signals - and keeps "no hit" reachable.
|
|
189
|
+
*
|
|
190
|
+
* WHY NORMALISE PER CHANNEL AND SUM, RATHER THAN RRF. Reciprocal rank fusion is the usual way to
|
|
191
|
+
* combine ranked lists, and it would be a regression here: it discards score magnitude, so a
|
|
192
|
+
* passage that matches decisively and one that matches barely would arrive at the same rank and
|
|
193
|
+
* finish within a couple of percent of each other. The gap between a clear winner and a near miss is
|
|
194
|
+
* exactly what BM25 was measuring. Each channel is therefore divided by its own best score for this
|
|
195
|
+
* query, which preserves the shape of that channel's confidence while making the weights comparable.
|
|
196
|
+
*/
|
|
197
|
+
search(workspaceId: string, query: string, options?: KnowledgeSearch): Promise<KnowledgeHit[]>;
|
|
198
|
+
/**
|
|
199
|
+
* Re-orders the candidate pool with the session's own model.
|
|
200
|
+
*
|
|
201
|
+
* WHY THIS EXISTS AND WHAT IT IS WORTH. Lexical retrieval ranks on shared characters, so a passage
|
|
202
|
+
* that answers in different words sits below passages that merely repeat the question's wording -
|
|
203
|
+
* often below the fold, where the character budget has already cut it off. Reading the candidates
|
|
204
|
+
* is something the model can already do, so it is asked to pick.
|
|
205
|
+
*
|
|
206
|
+
* ITS CEILING, STATED PLAINLY. Reranking can only reorder what the first stage returned. A query
|
|
207
|
+
* with NO lexical overlap at all never reaches this function, which is why the vector channel and
|
|
208
|
+
* not this one is what closes the paraphrase gap - see docs/KNOWLEDGE_BASE.md.
|
|
209
|
+
*
|
|
210
|
+
* WHY IT FAILS OPEN. Every outcome that is not a clean parse returns the lexical order. A reranker
|
|
211
|
+
* that made things worse, hung, or answered with prose would degrade the feature invisibly, and a
|
|
212
|
+
* retrieval layer that is allowed to get worse by being clever is worse than one that is not.
|
|
213
|
+
*/
|
|
214
|
+
private rerank;
|
|
215
|
+
/**
|
|
216
|
+
* Mints a read-only token for one turn. The agent can retrieve, never write: everything a person
|
|
217
|
+
* puts in a knowledge base is theirs to curate from the settings window.
|
|
218
|
+
*/
|
|
219
|
+
issue(conversationId: string): string;
|
|
220
|
+
release(token: string): void;
|
|
221
|
+
private identity;
|
|
222
|
+
/** The agent-facing endpoint: a title catalogue and a search. Nothing that writes. */
|
|
223
|
+
tool(token: string, body: {
|
|
224
|
+
kind?: string;
|
|
225
|
+
query?: string;
|
|
226
|
+
limit?: number;
|
|
227
|
+
}, signal?: AbortSignal): Promise<{
|
|
228
|
+
documents: {
|
|
229
|
+
title: string;
|
|
230
|
+
source: string;
|
|
231
|
+
characters: number;
|
|
232
|
+
}[];
|
|
233
|
+
hits?: undefined;
|
|
234
|
+
reranked?: undefined;
|
|
235
|
+
} | {
|
|
236
|
+
documents?: undefined;
|
|
237
|
+
hits: KnowledgeHit[];
|
|
238
|
+
reranked: boolean;
|
|
239
|
+
}>;
|
|
240
|
+
/** The browser-facing endpoint, behind the ordinary access token and same-origin check. */
|
|
241
|
+
manage(workspaceId: string, body: {
|
|
242
|
+
action?: string;
|
|
243
|
+
path?: string;
|
|
244
|
+
title?: string;
|
|
245
|
+
text?: string;
|
|
246
|
+
documentId?: string;
|
|
247
|
+
query?: string;
|
|
248
|
+
limit?: number;
|
|
249
|
+
}): Promise<{
|
|
250
|
+
hits: KnowledgeHit[];
|
|
251
|
+
documents: Omit<KnowledgeDocument, "text">[];
|
|
252
|
+
semantic: import("./knowledge-vectors.js").VectorStatus | null;
|
|
253
|
+
}>;
|
|
254
|
+
/**
|
|
255
|
+
* What the panel shows, including whether semantic retrieval is actually working.
|
|
256
|
+
*
|
|
257
|
+
* WHY THE STATE IS WORTH SHOWING. The vector half depends on an optional package and a download from
|
|
258
|
+
* a host the machine may not reach, and both fail quietly by design. Without this on screen, "the
|
|
259
|
+
* feature is off" is indistinguishable from "the feature is broken here" - which is the difference
|
|
260
|
+
* between not wanting a feature and not being able to have it.
|
|
261
|
+
*/
|
|
262
|
+
private overview;
|
|
263
|
+
}
|