@evandu/super-pi 0.1.83 → 0.1.85

package/dist/server/knowledge.js

@@ -0,0 +1,743 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
+import { basename, extname, join } from "node:path";
+import { looksBinary, resolveInsideRoot } from "./files.js";
+/** 1 MB. Big enough for a design doc or a spec, small enough that chunking stays instant. */
+export const MAX_DOCUMENT_BYTES = 1024 * 1024;
+/** A pasted document rides inside a JSON body, so the request cap leaves room for escaping overhead. */
+export const MAX_KNOWLEDGE_REQUEST_BYTES = MAX_DOCUMENT_BYTES * 2;
+/** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
+export const MAX_WORKSPACE_BYTES = 16 * 1024 * 1024;
+const CHUNK_CHARS = 1200;
+const CHUNK_OVERLAP_CHARS = 150;
+const MAX_CHUNKS_PER_DOCUMENT = 2000;
+export const DEFAULT_RESULT_LIMIT = 6;
+export const MAX_RESULT_LIMIT = 20;
+/** Per hit. Enough to answer from, short enough that six hits stay inside one turn's prompt budget. */
+const RESULT_TEXT_CAP = 1200;
+/** Across all hits of one search. A model that asked for 20 passages still gets a bounded prompt. */
+const RESULT_TOTAL_CAP = 12000;
+/** Below this many candidates there is no ranking to correct, and the model call is not worth it. */
+const RERANK_MIN_CANDIDATES = 3;
+/** How many candidates the model sees. Larger pools cost prompt without improving the top few. */
+const RERANK_POOL = 12;
+/** Per candidate, and the whole rerank. Both are ceilings on latency a tool call can add to a turn. */
+const RERANK_SNIPPET_CHARS = 240;
+const RERANK_TIMEOUT_MS = 12000;
+/**
+ * The first integer array in the model's answer, or null.
+ *
+ * WHY TOLERANT PARSING. The prompt asks for a bare JSON array, and a model that mostly complies will
+ * still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
+ * guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
+ * the lexical order.
+ */
+export function parseRanking(output) {
+    const match = /\[\s*\d+(?:\s*,\s*\d+)*\s*\]/.exec(output ?? "");
+    if (!match)
+        return null;
+    try {
+        const parsed = JSON.parse(match[0]);
+        if (!Array.isArray(parsed))
+            return null;
+        return parsed.filter((value) => Number.isInteger(value) && value > 0);
+    }
+    catch {
+        return null;
+    }
+}
+const CJK_RUN = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/gu;
+const LATIN_WORD = /[\p{L}\p{N}_.+#-]{2,}/gu;
+const HEADING = /^#{1,6}\s+(.*\S)\s*$/;
+/** The `source` of a document the person typed rather than imported. See `add` for why it matters. */
+const PASTED_SOURCE = "pasted";
+/** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
+export const KNOWLEDGE_TOOL_NAME = "knowledge_search";
+/** Extensions a document may be imported from. Mirrors `files.TEXT_EXTENSIONS`, minus the source languages. */
+const IMPORTABLE = new Set([
+    ".md", ".markdown", ".mdx", ".txt", ".text", ".log", ".rst", ".org",
+    ".json", ".jsonc", ".json5", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf",
+    ".csv", ".tsv", ".html", ".htm", ".xml", ".svg", ".sql", ".diff", ".patch",
+]);
+/**
+ * Index terms for one string: CJK bigrams plus Latin words.
+ *
+ * WHY BIGRAMS AND NOT WORDS. Chinese has no spaces, so the word-list tokenizer the rest of this
+ * server uses (`ActiveMemory.search`) returns one enormous "word" for a whole sentence and matches
+ * nothing usefully. Character bigrams need no dictionary and no dependency, and they carry enough
+ * precision that 知识库 and 记忆库 share 识库 but not the whole run - which is what makes a two-term
+ * overlap meaningful rather than accidental.
+ */
+export function tokenize(text) {
+    return cjkNgrams(text, 2);
+}
+/**
+ * The longer-range channel: character TRIGRAMS, for the passage that repeats a phrase verbatim.
+ *
+ * WHY A SECOND GRANULARITY AND NOT JUST BIGRAMS. Bigrams answer "do these share any two-character
+ * run", which is high recall but low precision - two unrelated Chinese sentences collide constantly on
+ * a shared pair. Trigrams only fire on a real three-character overlap, so when the user's question
+ * quotes the document's own wording this channel separates it sharply from the rest of the corpus.
+ * It is scored on its own and fused with the bigram channel rather than merged into it, because the
+ * two answer different questions and a term that is strong for one is usually weak for the other.
+ */
+export function tokenizePhrases(text) {
+    return cjkNgrams(text, 3);
+}
+function cjkNgrams(text, size) {
+    const lowered = text.toLocaleLowerCase();
+    const terms = [];
+    for (const run of lowered.match(CJK_RUN) ?? []) {
+        // A run shorter than the n-gram falls back to itself rather than contributing nothing: "端口" is
+        // a legitimate two-character term that a trigram tokenizer would otherwise silently drop.
+        if (run.length <= size)
+            terms.push(run);
+        else
+            for (let i = 0; i + size <= run.length; i++)
+                terms.push(run.slice(i, i + size));
+    }
+    // CJK is removed before the Latin pass, because `\p{L}` also matches Han characters: without this,
+    // every Chinese run would be indexed a second time as one long term.
+    for (const word of lowered.replace(CJK_RUN, " ").match(LATIN_WORD) ?? [])
+        terms.push(word);
+    return terms;
+}
+/** The identity of a document's CONTENT, independent of its name, its path and when it arrived. */
+export function digestOf(text) {
+    return createHash("sha256").update(text).digest("hex");
+}
+/**
+ * Splits the document into units that must not be pulled apart: a fenced code block, a table, and
+ * every ordinary line.
+ *
+ * WHY GROUPING HAPPENS BEFORE CHUNKING. A fence or a table is the case where a naive line-based
+ * splitter does real damage rather than cosmetic damage: cutting a 40-line SQL block in half produces
+ * two passages that are each syntactically meaningless, and neither of which a reader could have
+ * learned anything from - the retrieval then returns one of the halves and the model reasons from a
+ * fragment. Grouping first means such a block is only ever split when it alone exceeds a chunk.
+ */
+function units(text) {
+    const out = [];
+    let fence = "", buffer = [];
+    const flush = () => { if (buffer.length) {
+        out.push(buffer.join("\n"));
+        buffer = [];
+    } };
+    for (const line of text.split(/\r?\n/)) {
+        const marker = /^\s*(```+|~~~+)/.exec(line);
+        if (fence) {
+            buffer.push(line);
+            // A closing fence is one that uses the SAME character and is at least as long as the opener;
+            // a ``` inside a ~~~ block is content, not a terminator.
+            if (marker && line.trim()[0] === fence[0] && marker[1].length >= fence.length) {
+                fence = "";
+                flush();
+            }
+            continue;
+        }
+        if (marker) {
+            flush();
+            fence = marker[1];
+            buffer.push(line);
+            continue;
+        }
+        if (/^\s*\|/.test(line)) {
+            buffer.push(line);
+            continue;
+        }
+        flush();
+        out.push(line);
+    }
+    flush();
+    return out;
+}
+/**
+ * Splits a document into overlapping chunks, remembering the heading each one sits under.
+ *
+ * WHY OVERLAP. A passage that spans a boundary is the passage most likely to be the answer - the
+ * sentence that defines a term usually starts in one chunk and ends in the next - and a retrieval
+ * system that cut there would return a fragment ending mid-thought. The 150-character tail is
+ * repeated at the head of the next chunk to keep that sentence whole in at least one hit.
+ */
+export function chunkDocument(text) {
+    const chunks = [];
+    let heading = "", buffer = "", bufferHeading = "";
+    const flush = () => {
+        const trimmed = buffer.trim();
+        if (trimmed)
+            chunks.push({ index: chunks.length, heading: bufferHeading, text: trimmed });
+        buffer = "";
+    };
+    for (const unit of units(text)) {
+        const headingMatch = HEADING.exec(unit.trim());
+        // A heading starts a new chunk rather than ending one: a section is the unit a reader would cite.
+        if (headingMatch) {
+            flush();
+            heading = headingMatch[1].trim();
+            bufferHeading = heading;
+        }
+        // A UNIT LONGER THAN A CHUNK IS ITS OWN PROBLEM. Minified JSON, a fenced file too long to read in
+        // one go: nothing can keep it whole, so it is split by characters with the overlap preserved,
+        // rather than becoming one oversized chunk - which every search would then truncate to the
+        // per-passage cap, leaving a document that is indexed but effectively unreadable.
+        if (unit.length > CHUNK_CHARS) {
+            for (let offset = 0; offset < unit.length; offset += CHUNK_CHARS - CHUNK_OVERLAP_CHARS) {
+                bufferHeading = heading;
+                buffer = unit.slice(offset, offset + CHUNK_CHARS);
+                flush();
+            }
+            continue;
+        }
+        if (buffer.length + unit.length + 1 > CHUNK_CHARS) {
+            const overlap = buffer.slice(-CHUNK_OVERLAP_CHARS);
+            flush();
+            buffer = bufferHeading === heading ? overlap : "";
+        }
+        buffer += (buffer ? "\n" : "") + unit;
+    }
+    flush();
+    return chunks;
+}
+/** BM25's two knobs. Named, because "some other constants" is not a retrievable explanation. */
+const K1 = 1.2;
+const B = 0.75;
+/**
+ * How much each channel is worth, BEFORE normalisation.
+ *
+ * `document` outranks `head` on purpose. A question that names a document ("这个规范怎么规定的") is
+ * better answered by boosting every chunk of that document than by matching section titles, because
+ * the chunk that actually answers may sit in a section whose heading shares nothing with the question.
+ */
+const WEIGHT = { body: 1, phrase: 0.45, head: 0.5, document: 0.7, vector: 0.9 };
+function countTerms(terms) {
+    const counts = new Map();
+    for (const term of terms)
+        counts.set(term, (counts.get(term) ?? 0) + 1);
+    return counts;
+}
+/**
+ * Standard BM25 for one map against a query.
+ *
+ * `total` is the number of units being ranked (chunks, or documents for the document channel) and
+ * `df` how many of them contain each term - the two inputs that decide how rare a term is.
+ */
+function bm25(counts, length, averageLength, df, total, terms) {
+    let score = 0;
+    for (const term of terms) {
+        const frequency = counts.get(term);
+        if (!frequency)
+            continue;
+        const documentsWithTerm = df.get(term) ?? 0;
+        // The +1 inside the log keeps the idf positive for a term EVERY unit contains, which is otherwise
+        // a negative weight: a common term would subtract from the score of the very passage that repeats
+        // it, and BM25's ceiling would rank that passage below a worse one.
+        const idf = Math.log(1 + (total - documentsWithTerm + 0.5) / (documentsWithTerm + 0.5));
+        score += idf * (frequency * (K1 + 1)) / (frequency + K1 * (1 - B + B * (length / averageLength)));
+    }
+    return score;
+}
+export class KnowledgeBase {
+    directory;
+    store;
+    runtime;
+    vectors;
+    capabilities = new Map();
+    indexes = new Map();
+    serial = Promise.resolve();
+    constructor(directory, store, runtime, vectors) {
+        this.directory = directory;
+        this.store = store;
+        this.runtime = runtime;
+        this.vectors = vectors;
+    }
+    /** `SUPER_PI_KNOWLEDGE_ENABLED=0` turns the whole feature off, the same way memory turns off. */
+    enabled() { return process.env.SUPER_PI_KNOWLEDGE_ENABLED !== "0"; }
+    /** The opaque workspace identity on disk, encoded. Paths never come from the model or the request. */
+    folder(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
+    file(workspaceId, documentId) { return join(this.folder(workspaceId), `${documentId}.json`); }
+    requireWorkspace(workspaceId) {
+        if (!this.store.snapshot().workspaces.some(workspace => workspace.id === workspaceId))
+            throw new Error("工作区不存在");
+    }
+    async locked(work) {
+        const task = this.serial.catch(() => { }).then(work);
+        this.serial = task;
+        return task;
+    }
+    /** Metadata only - the caller listing documents must never receive the full text of a workspace. */
+    summarise(document) {
+        const { text: _text, ...meta } = document;
+        return meta;
+    }
+    async readAll(workspaceId) {
+        let names;
+        try {
+            names = await readdir(this.folder(workspaceId));
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return [];
+            throw error;
+        }
+        const documents = [];
+        for (const name of names) {
+            if (!name.endsWith(".json"))
+                continue;
+            try {
+                const document = JSON.parse(await readFile(join(this.folder(workspaceId), name), "utf8"));
+                // A half-written or hand-edited file must not take the whole workspace down with it.
+                if (typeof document?.id !== "string" || typeof document.text !== "string")
+                    continue;
+                // A document stored before fingerprints existed hashes on read. It is deliberately NOT
+                // written back: reading must not mutate, and the next import of that document rewrites it.
+                if (typeof document.digest !== "string")
+                    document.digest = digestOf(document.text);
+                documents.push(document);
+            }
+            catch (error) {
+                console.warn(`Super Pi: 跳过无法解析的知识库文档 ${name}：${error instanceof Error ? error.message : "unknown error"}`);
+            }
+        }
+        return documents;
+    }
+    async save(document) {
+        const folder = this.folder(document.workspaceId);
+        await mkdir(folder, { recursive: true });
+        const file = this.file(document.workspaceId, document.id), temporary = `${file}.${randomUUID()}.tmp`;
+        await writeFile(temporary, JSON.stringify(document), { encoding: "utf8", mode: 0o600 });
+        await rename(temporary, file);
+    }
+    index(document) {
+        const chunks = chunkDocument(document.text);
+        if (chunks.length > MAX_CHUNKS_PER_DOCUMENT)
+            throw new Error(`文档过长：切分后有 ${chunks.length} 个片段，超过 ${MAX_CHUNKS_PER_DOCUMENT} 的上限`);
+        const channels = [];
+        const docTerms = new Map();
+        let docLength = 0;
+        for (const chunk of chunks) {
+            // Tokenised ONCE here, at import time, rather than on every search: a workspace with a hundred
+            // documents would otherwise re-split the whole corpus per question.
+            const body = countTerms(tokenize(chunk.text));
+            const channelsForChunk = {
+                body,
+                phrase: countTerms(tokenizePhrases(chunk.text)),
+                // The document title is folded into every chunk's heading channel: it names the document in
+                // the citation the reader sees, so a question about the document should reach it.
+                head: countTerms(tokenize(`${document.title} ${chunk.heading}`)),
+                bodyLength: [...body.values()].reduce((sum, count) => sum + count, 0),
+            };
+            channels.push(channelsForChunk);
+            for (const [term, count] of body)
+                docTerms.set(term, (docTerms.get(term) ?? 0) + count);
+            docLength += channelsForChunk.bodyLength;
+        }
+        return { meta: this.summarise(document), chunks, channels, docTerms, docLength };
+    }
+    /**
+     * Loads (or rebuilds) one workspace's index. Cache miss is the only thing that reads the disk.
+     *
+     * WHY REBUILT PER WORKSPACE AND NOT ONE GLOBAL INDEX. Isolation is a security property here, not a
+     * performance one: a hit may only ever come from the workspace the turn's capability names. A
+     * single shared index would need every search to filter by workspace after scoring, which is one
+     * forgotten filter away from a cross-tenant leak. Separate caches cannot express that mistake.
+     *
+     * WHY THERE IS NO ON-DISK INDEX. Measured, not assumed: rebuilding the index of a 16 MB workspace -
+     * the feature's own capacity cap - costs about 180 ms, once per process. A serialised index would be
+     * larger than the corpus it describes and JSON.parse of it is not reliably cheaper than the
+     * tokenisation, so it would trade a correctness-neutral optimisation for a second on-disk artefact
+     * that can drift from the documents.
+     */
+    async load(workspaceId) {
+        const cached = this.indexes.get(workspaceId);
+        if (cached)
+            return cached;
+        const documents = (await this.readAll(workspaceId)).map(document => this.index(document));
+        const index = {
+            documents,
+            averageLength: documents.length ? documents.reduce((sum, document) => sum + document.docLength, 0) / documents.length : 0,
+        };
+        this.indexes.set(workspaceId, index);
+        // Vectors are built in the background on first sight of a workspace, so a restart with an
+        // already-indexed knowledge base costs nothing and one without pays for it while answering
+        // lexically rather than while the person waits.
+        this.vectors?.schedule(workspaceId, documents.map(document => ({ id: document.meta.id, digest: document.meta.digest, chunks: document.chunks })));
+        return index;
+    }
+    invalidate(workspaceId) { this.indexes.delete(workspaceId); }
+    async list(workspaceId) {
+        this.requireWorkspace(workspaceId);
+        const documents = (await this.readAll(workspaceId)).map(document => this.summarise(document));
+        return documents.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
+    }
+    /**
+     * Whether this workspace has any document at all, which is what decides whether the turn mounts the
+     * extension.
+     *
+     * WHY A DIRECTORY LISTING AND NOT `list()`. This runs once per turn, before every spawn. `list()`
+     * would read and parse every document's full text to answer a yes/no question, so a workspace with
+     * a dozen real documents would pay for parsing all of them on every message - including the
+     * catalogue fetch the extension makes the moment it loads. `readdir` answers the same question for
+     * the price of one directory entry, and a name is all this needs.
+     */
+    async hasDocuments(workspaceId) {
+        if (!this.enabled())
+            return false;
+        if (!this.store.snapshot().workspaces.some(workspace => workspace.id === workspaceId))
+            return false;
+        try {
+            return (await readdir(this.folder(workspaceId))).some(name => name.endsWith(".json"));
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return false;
+            throw error;
+        }
+    }
+    /**
+     * Adds a document, either from a workspace file or from pasted text.
+     *
+     * WHY THE FILE IS READ HERE AND NOT UPLOADED. A knowledge base is made of documents the person
+     * ALREADY has in their workspace - re-uploading a copy of the file would give the agent a second,
+     * newer version that diverges from the one the workspace serves. The path therefore goes through
+     * the same `resolveInsideRoot` guard the file editor uses, so symlink and `..` escapes are refused
+     * by construction instead of by a check someone has to remember.
+     */
+    async add(workspaceId, body) {
+        this.requireWorkspace(workspaceId);
+        if (!this.enabled())
+            throw new Error("知识库已关闭");
+        let text = typeof body.text === "string" ? body.text : undefined;
+        let source = PASTED_SOURCE, title = typeof body.title === "string" ? body.title.trim() : "", format = ".md";
+        if (text === undefined) {
+            if (typeof body.path !== "string" || !body.path.trim())
+                throw new Error("需要提供文件路径或正文");
+            const workspace = this.store.snapshot().workspaces.find(item => item.id === workspaceId);
+            const absolute = await resolveInsideRoot(workspace.path, body.path.trim());
+            const info = await stat(absolute);
+            if (!info.isFile())
+                throw new Error("只能导入文件");
+            format = extname(absolute).toLocaleLowerCase();
+            if (!IMPORTABLE.has(format))
+                throw new Error(`不支持的文件类型 ${format || "(无扩展名)"}；请先转换为 Markdown 或纯文本`);
+            if (info.size > MAX_DOCUMENT_BYTES)
+                throw new Error(`文件过大：${info.size} 字节超过 ${MAX_DOCUMENT_BYTES} 的上限`);
+            const buffer = await readFile(absolute);
+            if (looksBinary(buffer.subarray(0, 4096)))
+                throw new Error("看起来是二进制文件，无法索引");
+            text = buffer.toString("utf8");
+            source = body.path.trim();
+            if (!title)
+                title = basename(absolute);
+        }
+        const trimmed = text.trim();
+        if (!trimmed)
+            throw new Error("文档内容为空");
+        if (Buffer.byteLength(trimmed, "utf8") > MAX_DOCUMENT_BYTES)
+            throw new Error(`正文过大：超过 ${MAX_DOCUMENT_BYTES} 字节的上限`);
+        return this.locked(async () => {
+            const existing = await this.readAll(workspaceId);
+            const digest = digestOf(trimmed);
+            // The same CONTENT is already here, under any name or path. Importing it again would give the
+            // agent two copies that answer identically and make every search return the same passage twice.
+            const duplicate = existing.find(document => document.digest === digest);
+            if (duplicate)
+                return this.summarise(duplicate);
+            // The same PATH with DIFFERENT content is the user having edited the file and re-importing it.
+            // That is an update, not a second document: keeping both would leave the agent citing the
+            // version the person just replaced, with no way to tell which one is current.
+            //
+            // PASTED TEXT IS EXEMPT. `source` is the literal "pasted" for every hand-entered document, so
+            // without this exemption a second pasted note would match the first one's source and silently
+            // REPLACE it - two unrelated notes, one of them gone, with nothing in the response to say so.
+            const previous = source === PASTED_SOURCE ? undefined : existing.find(document => document.source === source && document.format === format);
+            const document = {
+                id: previous?.id ?? randomUUID(), workspaceId,
+                title: title.slice(0, 120) || "未命名文档", source, format, digest,
+                characters: trimmed.length,
+                createdAt: previous?.createdAt ?? new Date().toISOString(),
+                updatedAt: new Date().toISOString(), text: trimmed,
+            };
+            const used = existing.reduce((sum, document) => sum + document.characters, 0) - (previous?.characters ?? 0) + trimmed.length;
+            if (used > MAX_WORKSPACE_BYTES)
+                throw new Error(`工作区知识库容量不足：${used}/${MAX_WORKSPACE_BYTES} 字符；请删除不再需要的文档`);
+            // Chunked before it is persisted, so an unindexable document is refused rather than stored and
+            // then failing on every later search.
+            this.index(document);
+            await this.save(document);
+            // Vectors are keyed by content, so a replaced document's old vectors are dropped before the new
+            // ones are scheduled. Leaving them would let a search cite the pre-edit passage for the
+            // post-edit question.
+            await this.vectors?.forget(workspaceId, document.id);
+            this.invalidate(workspaceId);
+            this.vectors?.schedule(workspaceId, [{ id: document.id, digest: document.digest, chunks: chunkDocument(document.text) }]);
+            return this.summarise(document);
+        });
+    }
+    async remove(workspaceId, documentId) {
+        this.requireWorkspace(workspaceId);
+        return this.locked(async () => {
+            // Existence is checked by reading the folder rather than trusting the id: an id that does not
+            // belong to THIS workspace must not be able to delete anything, and that is decided by the path.
+            const documents = await this.readAll(workspaceId);
+            if (!documents.some(document => document.id === documentId))
+                throw new Error("知识库文档不存在");
+            await unlink(this.file(workspaceId, documentId));
+            await this.vectors?.forget(workspaceId, documentId);
+            this.invalidate(workspaceId);
+        });
+    }
+    /**
+     * Scores this workspace's chunks on four independent lexical channels and fuses them.
+     *
+     * WHY LEXICAL AND NOT ONLY EMBEDDINGS. See docs/KNOWLEDGE_BASE.md: the vector channel is fused in
+     * here rather than replacing this one, because each is wrong in a way the other catches. BM25
+     * misses a paraphrase completely; a vector search will happily return the nearest thing at a
+     * confident-looking 0.58 when the answer is not in the knowledge base at all. Summing a normalised
+     * lexical score with a normalised cosine score keeps both signals - and keeps "no hit" reachable.
+     *
+     * WHY NORMALISE PER CHANNEL AND SUM, RATHER THAN RRF. Reciprocal rank fusion is the usual way to
+     * combine ranked lists, and it would be a regression here: it discards score magnitude, so a
+     * passage that matches decisively and one that matches barely would arrive at the same rank and
+     * finish within a couple of percent of each other. The gap between a clear winner and a near miss is
+     * exactly what BM25 was measuring. Each channel is therefore divided by its own best score for this
+     * query, which preserves the shape of that channel's confidence while making the weights comparable.
+     */
+    async search(workspaceId, query, options = {}) {
+        const wanted = Math.min(Math.max(Math.trunc(options.limit ?? DEFAULT_RESULT_LIMIT) || DEFAULT_RESULT_LIMIT, 1), MAX_RESULT_LIMIT);
+        const terms = [...new Set(tokenize(query ?? ""))];
+        const phrases = [...new Set(tokenizePhrases(query ?? ""))];
+        if (!terms.length && !phrases.length)
+            return [];
+        const { documents, averageLength } = await this.load(workspaceId);
+        if (!documents.length || !averageLength)
+            return [];
+        const chunkCount = documents.reduce((sum, document) => sum + document.chunks.length, 0);
+        // Document frequency is counted once per channel, over that channel's own unit: chunks for the
+        // passage channels, documents for the document channel.
+        const chunkDf = new Map(), phraseDf = new Map(), headDf = new Map(), docDf = new Map();
+        for (const document of documents) {
+            for (const channels of document.channels) {
+                for (const term of terms) {
+                    if (channels.body.has(term))
+                        chunkDf.set(term, (chunkDf.get(term) ?? 0) + 1);
+                    if (channels.head.has(term))
+                        headDf.set(term, (headDf.get(term) ?? 0) + 1);
+                }
+                for (const phrase of phrases)
+                    if (channels.phrase.has(phrase))
+                        phraseDf.set(phrase, (phraseDf.get(phrase) ?? 0) + 1);
+            }
+            for (const term of terms)
+                if (document.docTerms.has(term))
+                    docDf.set(term, (docDf.get(term) ?? 0) + 1);
+        }
+        const hits = [];
+        let bodyBest = 0, phraseBest = 0, headBest = 0, docBest = 0;
+        for (const document of documents) {
+            // The document channel scores the DOCUMENT once and is then broadcast to all of its chunks: a
+            // question that names a document should reach the paragraph answering it, which may sit under a
+            // heading sharing no word with the question.
+            const documentScore = bm25(document.docTerms, document.docLength, averageLength, docDf, documents.length, terms);
+            if (documentScore > docBest)
+                docBest = documentScore;
+            document.channels.forEach((channels, position) => {
+                const chunk = document.chunks[position];
+                const body = bm25(channels.body, channels.bodyLength, averageLength, chunkDf, chunkCount, terms);
+                const phrase = bm25(channels.phrase, channels.bodyLength, averageLength, phraseDf, chunkCount, phrases);
+                const head = bm25(channels.head, channels.bodyLength, averageLength, headDf, chunkCount, terms);
+                if (body > bodyBest)
+                    bodyBest = body;
+                if (phrase > phraseBest)
+                    phraseBest = phrase;
+                if (head > headBest)
+                    headBest = head;
+                hits.push({
+                    hit: { documentId: document.meta.id, title: document.meta.title, source: document.meta.source, heading: chunk.heading, chunk: chunk.index, score: 0, text: chunk.text },
+                    // Stable across index rebuilds, so vectors written by a previous run still line up.
+                    key: `${document.meta.id}#${chunk.index}`,
+                    channels: { body, phrase, head, document: documentScore },
+                });
+            });
+        }
+        const lexical = hits.map(({ hit, key, channels }) => {
+            // A chunk is a LEXICAL candidate if it matched on its own terms - body, phrase or heading. The
+            // document channel only reweights candidates; it never creates one. Without this rule every
+            // chunk of a matching document becomes a "hit" on the strength of the document alone, which
+            // pads the pool with passages that mention nothing in the question, forces a rerank on
+            // searches that never needed one, and spends away the headroom that makes "no hit" visible.
+            const own = WEIGHT.body * (bodyBest ? channels.body / bodyBest : 0)
+                + WEIGHT.phrase * (phraseBest ? channels.phrase / phraseBest : 0)
+                + WEIGHT.head * (headBest ? channels.head / headBest : 0);
+            return { hit, key, own, lexical: own > 0 ? own : 0, document: channels.document };
+        });
+        // THE VECTOR CHANNEL, FUSED RATHER THAN SUBSTITUTED, and scored over EVERY chunk rather than over
+        // the lexical candidates - the chunks that share no characters with the question are precisely the
+        // ones the vector channel exists to find, so narrowing to lexical hits first would throw away the
+        // entire point. The two signals fail in opposite directions, which is the argument for both: the
+        // lexical channels miss a paraphrase completely, and a vector search returns the nearest thing at
+        // a confident-looking 0.58 when the answer is not in the knowledge base at all. The cosine keeps
+        // its own scale instead of being divided by the best hit, so "nothing here is close" survives as
+        // a low score rather than being rescaled into confidence.
+        const similarities = this.vectors
+            ? await this.vectors.search(workspaceId, query ?? "", lexical.map(entry => [entry.key, entry.hit.documentId]))
+            : null;
+        const fused = lexical
+            .map(entry => {
+            const similarity = similarities?.get(entry.key) ?? 0;
+            const score = Math.round(1000 * (entry.lexical + (similarities ? WEIGHT.vector * similarity : 0)
+                + (entry.lexical > 0 ? WEIGHT.document * (docBest ? entry.document / docBest : 0) : 0))) / 1000;
+            return { ...entry.hit, score };
+        })
+            .filter(hit => hit.score > 0);
+        // Best-first, with the tie broken by chunk order so two equal passages of one document come back
+        // in the order a reader would meet them.
+        fused.sort((a, b) => b.score - a.score || a.chunk - b.chunk);
+        const ordered = options.rerank ? await this.rerank(options.conversation, query ?? "", fused, options.signal) ?? fused : fused;
+        const selected = [];
+        let spent = 0;
+        for (const hit of ordered) {
+            if (selected.length >= wanted)
+                break;
+            const room = RESULT_TOTAL_CAP - spent;
+            if (room <= 0)
+                break;
+            // Spent in score order, so a budget that runs out drops the WORST hits rather than whichever
+            // ones happened to be visited last.
+            const text = hit.text.slice(0, Math.min(room, RESULT_TEXT_CAP));
+            if (!text)
+                break;
+            selected.push({ ...hit, text });
+            spent += text.length;
+        }
+        return selected;
+    }
+    /**
+     * Re-orders the candidate pool with the session's own model.
+     *
+     * WHY THIS EXISTS AND WHAT IT IS WORTH. Lexical retrieval ranks on shared characters, so a passage
+     * that answers in different words sits below passages that merely repeat the question's wording -
+     * often below the fold, where the character budget has already cut it off. Reading the candidates
+     * is something the model can already do, so it is asked to pick.
+     *
+     * ITS CEILING, STATED PLAINLY. Reranking can only reorder what the first stage returned. A query
+     * with NO lexical overlap at all never reaches this function, which is why the vector channel and
+     * not this one is what closes the paraphrase gap - see docs/KNOWLEDGE_BASE.md.
+     *
+     * WHY IT FAILS OPEN. Every outcome that is not a clean parse returns the lexical order. A reranker
+     * that made things worse, hung, or answered with prose would degrade the feature invisibly, and a
+     * retrieval layer that is allowed to get worse by being clever is worse than one that is not.
+     */
+    async rerank(conversation, query, candidates, signal) {
+        if (!conversation || !this.runtime || candidates.length < RERANK_MIN_CANDIDATES || candidates.length > MAX_RESULT_LIMIT)
+            return null;
+        if (process.env.SUPER_PI_KNOWLEDGE_RERANK === "0")
+            return null;
+        const pool = candidates.slice(0, RERANK_POOL);
+        const workspace = this.store.snapshot().workspaces.find(item => item.id === conversation.workspaceId);
+        if (!workspace)
+            return null;
+        const controller = new AbortController();
+        const cancel = () => controller.abort();
+        signal?.addEventListener("abort", cancel, { once: true });
+        const timer = setTimeout(cancel, RERANK_TIMEOUT_MS);
+        timer.unref();
+        try {
+            // A FRESH session id, and one that no capability is minted for: this is the server asking the
+            // model a question, not a user turn, and it must never appear in a transcript or inherit a
+            // conversation's tools.
+            const isolated = {
+                ...structuredClone(conversation), id: `knowledge-rerank-${conversation.id}-${randomUUID()}`,
+                agentId: undefined, messages: [], status: "idle",
+            };
+            const listing = pool.map((hit, position) => `[${position + 1}] 《${hit.title}》${hit.heading ? ` > ${hit.heading}` : ""}\n${hit.text.slice(0, RERANK_SNIPPET_CHARS).replace(/\n+/g, " ")}`).join("\n\n");
+            const prompt = `你是检索排序器。用户提出了一个问题，下面是知识库检索出的候选片段。请按「能否回答这个问题」从高到低排序，只输出最相关的前几个片段编号，输出 JSON 数组，例如 [3,1]。不要输出解释文字，不要输出编号以外的内容。\n\n问题：${query}\n\n候选片段：\n${listing}`;
+            let output = "";
+            for await (const piece of this.runtime.answer({ conversation: isolated, prompt, cwd: workspace.path, signal: controller.signal, knowledgeRerank: true })) {
+                output += piece;
+                if (controller.signal.aborted)
+                    return null;
+            }
+            const parsed = parseRanking(output);
+            if (!parsed?.length)
+                return null;
+            const picked = parsed.map((index) => pool[index - 1]).filter((hit) => Boolean(hit));
+            if (!picked.length)
+                return null;
+            // Anything the model left out keeps its lexical position after the picks, so a partial or
+            // truncated answer still returns the full pool rather than silently losing candidates.
+            const chosen = new Set(picked);
+            return [...picked, ...pool.filter(hit => !chosen.has(hit))];
+        }
+        catch {
+            return null;
+        }
+        finally {
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", cancel);
+        }
+    }
+    /**
+     * Mints a read-only token for one turn. The agent can retrieve, never write: everything a person
+     * puts in a knowledge base is theirs to curate from the settings window.
+     */
+    issue(conversationId) {
+        const conversation = this.store.snapshot().conversations.find(item => item.id === conversationId);
+        if (!conversation)
+            throw new Error("知识库源会话不存在");
+        const token = randomUUID();
+        this.capabilities.set(token, { conversationId, workspaceId: conversation.workspaceId });
+        return token;
+    }
+    release(token) { this.capabilities.delete(token); }
+    identity(token) {
+        const identity = this.capabilities.get(token);
+        // The workspace is re-checked against the store, so a token minted before a workspace was deleted
+        // cannot keep serving its documents afterwards.
+        if (!identity || !this.store.snapshot().workspaces.some(workspace => workspace.id === identity.workspaceId))
+            throw new Error("知识库授权已失效");
+        return identity;
+    }
+    /** The agent-facing endpoint: a title catalogue and a search. Nothing that writes. */
+    async tool(token, body, signal) {
+        const identity = this.identity(token);
+        if (!this.enabled())
+            throw new Error("知识库已关闭");
+        if (body.kind === "catalogue")
+            return { documents: (await this.list(identity.workspaceId)).map(document => ({ title: document.title, source: document.source, characters: document.characters })) };
+        if (body.kind !== "search")
+            throw new Error("未知知识库操作");
+        // The reranker needs the conversation it is ranking FOR: that is whose model answers, whose
+        // workspace is being searched, and whose turn this latency is charged to.
+        const conversation = this.store.snapshot().conversations.find(item => item.id === identity.conversationId);
+        const hits = await this.search(identity.workspaceId, body.query ?? "", { limit: body.limit, rerank: true, conversation, signal });
+        return { hits, reranked: hits.length > 0 && conversation !== undefined };
+    }
+    /** The browser-facing endpoint, behind the ordinary access token and same-origin check. */
+    async manage(workspaceId, body) {
+        this.requireWorkspace(workspaceId);
+        if (body.action === "add") {
+            await this.add(workspaceId, body);
+            return { ...await this.overview(workspaceId), hits: [] };
+        }
+        if (body.action === "remove") {
+            await this.remove(workspaceId, String(body.documentId ?? ""));
+            return { ...await this.overview(workspaceId), hits: [] };
+        }
+        // No `rerank` here: the settings preview is a peek at what is indexed, and it must answer while
+        // the person is still typing rather than spend a model call on a query they are about to change.
+        if (body.action === "search")
+            return { ...await this.overview(workspaceId), hits: await this.search(workspaceId, String(body.query ?? ""), { limit: body.limit }) };
+        return { ...await this.overview(workspaceId), hits: [] };
+    }
+    /**
+     * What the panel shows, including whether semantic retrieval is actually working.
+     *
+     * WHY THE STATE IS WORTH SHOWING. The vector half depends on an optional package and a download from
+     * a host the machine may not reach, and both fail quietly by design. Without this on screen, "the
+     * feature is off" is indistinguishable from "the feature is broken here" - which is the difference
+     * between not wanting a feature and not being able to have it.
+     */
+    async overview(workspaceId) {
+        return { documents: await this.list(workspaceId), semantic: this.vectors ? this.vectors.status(await this.vectors.count(workspaceId)) : null };
+    }
+}