@evandu/super-pi 0.1.83 → 0.1.85

package/dist/server/knowledge-vectors.d.ts

@@ -0,0 +1,133 @@
+/**
+ * The model, and the only one this feature can use.
+ *
+ * WHY THIS ONE. `bge-small-zh` scores 0.618 nDCG@10 averaged over the Chinese retrieval tasks it is
+ * benchmarked on - against 0.695 for `bge-base-zh`, which is four times the download. `gte-multilingual-base`
+ * scores higher on both Chinese and English, at 324 MB against this one's 23 MB. For a workspace-sized
+ * corpus the next thing that would help is not a bigger model but fusing this one with the lexical
+ * channels, which costs nothing and is what `KnowledgeBase.search` does.
+ */
+export declare const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
+/**
+ * HOW BIG ONE EMBEDDED PIECE MAY BE, in characters.
+ *
+ * THIS NUMBER IS A MODEL LIMIT, NOT A TUNING KNOB, and getting it wrong is invisible. The model
+ * declares `max_position_embeddings: 512`, and a Chinese character costs roughly one token, so a piece
+ * is a little under half the window: 300 characters leaves room for the `[CLS]`/`[SEP]` pair and for
+ * tokenisation that splits a few characters into two. The retrieval chunks are 1200 characters - more
+ * than double this - which is fine for something a MODEL reads and fatal for something a MODEL
+ * ENCODES: handed a 1200-character chunk, roughly 60% of it is silently discarded past the window.
+ * English would survive it, so the degradation would land only on Chinese: the language this project
+ * is written in.
+ */
+export declare const EMBED_PIECE_CHARS = 300;
+export type VectorRecord = {
+    chunk: number;
+    values: number[];
+};
+/** What the settings panel shows, so "off" and "broken here" are never the same silent nothing. */
+export type VectorStatus = {
+    enabled: boolean;
+    ready: boolean;
+    indexed: number;
+    error?: string;
+    bundled?: boolean;
+};
+/**
+ * Splits a retrieval chunk into pieces small enough for the model's window.
+ *
+ * WHY NOT SHRINK THE RETRIEVAL CHUNKS INSTEAD. 1200 characters is the right size for a passage a model
+ * READS - it holds enough context to answer from. Encoding is a different constraint from reading, and
+ * conflating them would trade retrieval quality for a convenience the encoder does not need.
+ */
+export declare function embedPieces(text: string): string[];
+/** Element-wise max over the pieces, then unit length. See `pool` for why max and not mean. */
+export declare function pool(vectors: number[][]): number[];
+export declare function cosine(a: number[], b: number[]): number;
+/** One document's vectors, on disk next to the document they describe. */
+export type VectorDocument = {
+    digest: string;
+    records: VectorRecord[];
+};
+/**
+ * The optional vector half of retrieval.
+ *
+ * EVERY FAILURE HERE IS A FALLBACK TO LEXICAL, NEVER AN ERROR. The model is an optional npm package
+ * that a skipped optional install can take away, and it is a 512-token window
+ * wrapped around a Chinese corpus. Any of those failing is a reason to answer from the lexical
+ * channels and say so - never a reason to fail a search that the lexical channels can already serve.
+ */
+export declare class VectorIndex {
+    private readonly directory;
+    private extractor?;
+    private loading?;
+    private failure?;
+    private cache;
+    private pending;
+    private queue;
+    private background?;
+    constructor(directory: string);
+    /**
+     * Where the shipped weights live, relative to this module.
+     *
+     * `dist/server/knowledge-vectors.js` is two levels below the package root both in a published
+     * package and in this repository's own build - the same `../../assets/...` shape `runtime.ts` uses to
+     * reach `agents/`. The directory keeps the upstream id in its path because that is how the library
+     * looks the model up under `localModelPath`.
+     */
+    bundledModel(): string;
+    enabled(): boolean;
+    status(indexed: number): VectorStatus;
+    private workspace;
+    private folder;
+    private file;
+    /**
+     * Loads the model once, from the copy that ships with this package. The `dtype` is not optional: on
+     * Node the library defaults to fp32, which for this model is 90 MB rather than 23.
+     *
+     * THERE IS NO NETWORK HERE, and that is the whole point of vendoring the weights. The first version
+     * downloaded the model on first use, which put a 23 MB fetch behind a host the machine may not be
+     * able to reach, on the first search, for every person who installed the package. Shipping the
+     * weights turns a failure mode into a file, and makes the feature work on a machine that has never
+     * been online since it was installed.
+     */
+    private load;
+    private embed;
+    /** Vectors already stored for one workspace, keyed by document then chunk index. */
+    private read;
+    count(workspaceId: string): Promise<number>;
+    /** Drops a document's vectors. Called whenever its content changes or it is deleted. */
+    forget(workspaceId: string, documentId: string): Promise<void>;
+    /**
+     * Embeds anything not already stored, WITHOUT waiting for it.
+     *
+     * WHY IT IS NOT AWAITED. This runs after an import or on the first search of a session, and the
+     * first run of it downloads a model. Neither should make a person wait: the import returns as soon
+     * as the document is stored and the search returns lexical hits, and a later search is simply better
+     * because by then the vectors exist.
+     *
+     * WHY REQUESTS QUEUE RATHER THAN DROP. One job runs at a time - two would load the same model twice
+     * and race each other's writes - but a request arriving mid-job has to be remembered. Dropping it
+     * would mean a document imported while a background index was in flight stayed unindexed until the
+     * next restart, which is the kind of bug that only shows up on a busy machine weeks later.
+     */
+    schedule(workspaceId: string, documents: Array<{
+        id: string;
+        digest: string;
+        chunks: Array<{
+            index: number;
+            text: string;
+        }>;
+    }>): void;
+    /**
+     * Cosine of every stored chunk against the query, best first.
+     *
+     * WHY THE RAW COSINE IS NOT RE-NORMALISED BY THE BEST HIT. Every lexical channel is divided by its
+     * own maximum, and that is right for them because BM25 scores are unbounded and have no absolute
+     * meaning. Cosine is the opposite: it is already a bounded, interpretable quantity, and dividing it
+     * by the best hit of the query would turn "nothing here is close" into "the best one is 1.0". The
+     * knowledge base's most useful property is that it can say it does not know, and rescaling is exactly
+     * how that gets spent.
+     */
+    search(workspaceId: string, query: string, keys: Array<[string, string]>): Promise<Map<string, number> | null>;
+}

package/dist/server/knowledge-vectors.js

@@ -0,0 +1,302 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+/**
+ * The model, and the only one this feature can use.
+ *
+ * WHY THIS ONE. `bge-small-zh` scores 0.618 nDCG@10 averaged over the Chinese retrieval tasks it is
+ * benchmarked on - against 0.695 for `bge-base-zh`, which is four times the download. `gte-multilingual-base`
+ * scores higher on both Chinese and English, at 324 MB against this one's 23 MB. For a workspace-sized
+ * corpus the next thing that would help is not a bigger model but fusing this one with the lexical
+ * channels, which costs nothing and is what `KnowledgeBase.search` does.
+ */
+export const KNOWLEDGE_MODEL = "Xenova/bge-small-zh-v1.5";
+/**
+ * HOW BIG ONE EMBEDDED PIECE MAY BE, in characters.
+ *
+ * THIS NUMBER IS A MODEL LIMIT, NOT A TUNING KNOB, and getting it wrong is invisible. The model
+ * declares `max_position_embeddings: 512`, and a Chinese character costs roughly one token, so a piece
+ * is a little under half the window: 300 characters leaves room for the `[CLS]`/`[SEP]` pair and for
+ * tokenisation that splits a few characters into two. The retrieval chunks are 1200 characters - more
+ * than double this - which is fine for something a MODEL reads and fatal for something a MODEL
+ * ENCODES: handed a 1200-character chunk, roughly 60% of it is silently discarded past the window.
+ * English would survive it, so the degradation would land only on Chinese: the language this project
+ * is written in.
+ */
+export const EMBED_PIECE_CHARS = 300;
+const EMBED_PIECE_OVERLAP = 40;
+/** Per query. Cosine similarity of unit vectors; beyond this the passage is noise. */
+const MIN_COSINE = 0.15;
+/**
+ * Splits a retrieval chunk into pieces small enough for the model's window.
+ *
+ * WHY NOT SHRINK THE RETRIEVAL CHUNKS INSTEAD. 1200 characters is the right size for a passage a model
+ * READS - it holds enough context to answer from. Encoding is a different constraint from reading, and
+ * conflating them would trade retrieval quality for a convenience the encoder does not need.
+ */
+export function embedPieces(text) {
+    const pieces = [];
+    for (let offset = 0; offset < text.length; offset += EMBED_PIECE_CHARS - EMBED_PIECE_OVERLAP) {
+        const piece = text.slice(offset, offset + EMBED_PIECE_CHARS).trim();
+        if (piece)
+            pieces.push(piece);
+        if (offset + EMBED_PIECE_CHARS >= text.length)
+            break;
+    }
+    return pieces.length ? pieces : [text.slice(0, EMBED_PIECE_CHARS)];
+}
+/** Element-wise max over the pieces, then unit length. See `pool` for why max and not mean. */
+export function pool(vectors) {
+    if (!vectors.length)
+        return [];
+    const pooled = new Array(vectors[0].length).fill(Number.NEGATIVE_INFINITY);
+    for (const vector of vectors) {
+        for (let i = 0; i < vector.length; i++)
+            if (vector[i] > pooled[i])
+                pooled[i] = vector[i];
+    }
+    let norm = 0;
+    for (const value of pooled)
+        norm += value * value;
+    norm = Math.sqrt(norm) || 1;
+    return pooled.map(value => value / norm);
+}
+export function cosine(a, b) {
+    let dot = 0;
+    for (let i = 0; i < a.length && i < b.length; i++)
+        dot += a[i] * b[i];
+    return dot;
+}
+/**
+ * The optional vector half of retrieval.
+ *
+ * EVERY FAILURE HERE IS A FALLBACK TO LEXICAL, NEVER AN ERROR. The model is an optional npm package
+ * that a skipped optional install can take away, and it is a 512-token window
+ * wrapped around a Chinese corpus. Any of those failing is a reason to answer from the lexical
+ * channels and say so - never a reason to fail a search that the lexical channels can already serve.
+ */
+export class VectorIndex {
+    directory;
+    extractor;
+    loading;
+    failure;
+    cache = new Map();
+    pending = new Set();
+    queue = [];
+    background;
+    constructor(directory) {
+        this.directory = directory;
+    }
+    /**
+     * Where the shipped weights live, relative to this module.
+     *
+     * `dist/server/knowledge-vectors.js` is two levels below the package root both in a published
+     * package and in this repository's own build - the same `../../assets/...` shape `runtime.ts` uses to
+     * reach `agents/`. The directory keeps the upstream id in its path because that is how the library
+     * looks the model up under `localModelPath`.
+     */
+    bundledModel() { return resolve(import.meta.dirname, "../../assets/knowledge-model"); }
+    enabled() { return process.env.SUPER_PI_KNOWLEDGE_EMBEDDINGS !== "0"; }
+    status(indexed) {
+        return {
+            enabled: this.enabled(), ready: Boolean(this.extractor), indexed, error: this.failure,
+            // Whether the shipped weights are on disk, reported before the model is ever loaded: "the
+            // package was built without its assets" and "the model failed to load" are different problems.
+            bundled: existsSync(this.bundledModel()),
+        };
+    }
+    workspace(workspaceId) { return join(this.directory, Buffer.from(workspaceId).toString("hex")); }
+    folder(workspaceId) { return join(this.workspace(workspaceId), "vectors"); }
+    file(workspaceId, documentId) { return join(this.folder(workspaceId), `${documentId}.vec.json`); }
+    /**
+     * Loads the model once, from the copy that ships with this package. The `dtype` is not optional: on
+     * Node the library defaults to fp32, which for this model is 90 MB rather than 23.
+     *
+     * THERE IS NO NETWORK HERE, and that is the whole point of vendoring the weights. The first version
+     * downloaded the model on first use, which put a 23 MB fetch behind a host the machine may not be
+     * able to reach, on the first search, for every person who installed the package. Shipping the
+     * weights turns a failure mode into a file, and makes the feature work on a machine that has never
+     * been online since it was installed.
+     */
+    load() {
+        if (this.extractor)
+            return Promise.resolve(this.extractor);
+        if (this.failure)
+            return Promise.resolve(undefined);
+        this.loading ??= (async () => {
+            try {
+                if (!existsSync(this.bundledModel()))
+                    throw new Error(`随包分发的模型缺失：${this.bundledModel()}`);
+                // THE SPECIFIER GOES THROUGH A VARIABLE, and that is deliberate. Written as a literal,
+                // `tsc` resolves it while compiling - so a machine whose optional install skipped this package
+                // could not build the server at all, which makes an "optional" dependency mandatory in the
+                // only sense that matters to a person who installed it. Node resolves the variable at RUN time
+                // instead, where its absence is already handled: the throw below is the same branch that
+                // handles a missing model.
+                const specifier = "@huggingface/transformers";
+                const { pipeline, env } = await import(specifier);
+                // The library resolves `<localModelPath>/<model id>/onnx/...`, which is why the vendored tree
+                // keeps the `Xenova/bge-small-zh-v1.5` directory rather than being flattened.
+                env.localModelPath = this.bundledModel();
+                env.allowRemoteModels = false;
+                this.extractor = await pipeline("feature-extraction", KNOWLEDGE_MODEL, { dtype: "q8" });
+                return this.extractor;
+            }
+            catch (error) {
+                this.failure = error instanceof Error ? error.message : String(error);
+                console.warn(`Super Pi: 语义检索不可用，将只用关键词检索：${this.failure}`);
+                return undefined;
+            }
+            finally {
+                this.loading = undefined;
+            }
+        })();
+        return this.loading;
+    }
+    async embed(texts) {
+        if (!texts.length || !this.enabled())
+            return null;
+        const extractor = await this.load();
+        if (!extractor)
+            return null;
+        try {
+            const output = await extractor(texts, { pooling: "cls", normalize: true });
+            const dimension = output.dims[output.dims.length - 1];
+            return Array.from({ length: output.dims[0] }, (_, row) => Array.from(output.data.slice(row * dimension, (row + 1) * dimension)));
+        }
+        catch (error) {
+            this.failure = error instanceof Error ? error.message : String(error);
+            console.warn(`Super Pi: 生成向量失败，将只用关键词检索：${this.failure}`);
+            return null;
+        }
+    }
+    /** Vectors already stored for one workspace, keyed by document then chunk index. */
+    async read(workspaceId, documentId) {
+        try {
+            const document = JSON.parse(await readFile(this.file(workspaceId, documentId), "utf8"));
+            return Array.isArray(document?.records) ? document : null;
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return null;
+            return null;
+        }
+    }
+    async count(workspaceId) {
+        try {
+            const names = (await readdir(this.folder(workspaceId))).filter(name => name.endsWith(".vec.json"));
+            let total = 0;
+            for (const name of names) {
+                const document = await this.read(workspaceId, name.slice(0, -".vec.json".length));
+                total += document?.records.length ?? 0;
+            }
+            return total;
+        }
+        catch {
+            return 0;
+        }
+    }
+    /** Drops a document's vectors. Called whenever its content changes or it is deleted. */
+    async forget(workspaceId, documentId) {
+        this.cache.delete(`${workspaceId}/${documentId}`);
+        await unlink(this.file(workspaceId, documentId)).catch(() => { });
+    }
+    /**
+     * Embeds anything not already stored, WITHOUT waiting for it.
+     *
+     * WHY IT IS NOT AWAITED. This runs after an import or on the first search of a session, and the
+     * first run of it downloads a model. Neither should make a person wait: the import returns as soon
+     * as the document is stored and the search returns lexical hits, and a later search is simply better
+     * because by then the vectors exist.
+     *
+     * WHY REQUESTS QUEUE RATHER THAN DROP. One job runs at a time - two would load the same model twice
+     * and race each other's writes - but a request arriving mid-job has to be remembered. Dropping it
+     * would mean a document imported while a background index was in flight stayed unindexed until the
+     * next restart, which is the kind of bug that only shows up on a busy machine weeks later.
+     */
+    schedule(workspaceId, documents) {
+        if (!this.enabled() || !documents.length)
+            return;
+        this.queue.push({ workspaceId, documents });
+        if (this.background)
+            return;
+        this.background = (async () => {
+            while (this.queue.length) {
+                const next = this.queue.shift();
+                try {
+                    for (const document of next.documents) {
+                        const key = `${next.workspaceId}/${document.id}`;
+                        if (this.pending.has(key))
+                            continue;
+                        this.pending.add(key);
+                        const stored = await this.read(next.workspaceId, document.id);
+                        if (stored?.digest === document.digest && stored.records.length === document.chunks.length) {
+                            this.pending.delete(key);
+                            continue;
+                        }
+                        const vectors = [];
+                        for (const chunk of document.chunks) {
+                            const embedded = await this.embed(embedPieces(chunk.text));
+                            if (!embedded) {
+                                this.pending.delete(key);
+                                return;
+                            }
+                            vectors.push({ chunk: chunk.index, values: pool(embedded) });
+                        }
+                        const folder = this.folder(next.workspaceId);
+                        await mkdir(folder, { recursive: true });
+                        const file = this.file(next.workspaceId, document.id), temporary = `${file}.${randomUUID()}.tmp`;
+                        await writeFile(temporary, JSON.stringify({ digest: document.digest, records: vectors }), { encoding: "utf8", mode: 0o600 });
+                        await rename(temporary, file);
+                        this.cache.set(key, new Map(vectors.map(record => [record.chunk, record])));
+                        this.pending.delete(key);
+                    }
+                }
+                catch (error) {
+                    console.warn(`Super Pi: 语义索引失败，将只用关键词检索：${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+        })().finally(() => { this.background = undefined; });
+    }
+    /**
+     * Cosine of every stored chunk against the query, best first.
+     *
+     * WHY THE RAW COSINE IS NOT RE-NORMALISED BY THE BEST HIT. Every lexical channel is divided by its
+     * own maximum, and that is right for them because BM25 scores are unbounded and have no absolute
+     * meaning. Cosine is the opposite: it is already a bounded, interpretable quantity, and dividing it
+     * by the best hit of the query would turn "nothing here is close" into "the best one is 1.0". The
+     * knowledge base's most useful property is that it can say it does not know, and rescaling is exactly
+     * how that gets spent.
+     */
+    async search(workspaceId, query, keys) {
+        if (!this.enabled() || !keys.length)
+            return null;
+        const documentIds = new Set(keys.map(([, documentId]) => documentId));
+        // First query of the session: read what was indexed last time rather than assuming nothing is.
+        for (const documentId of documentIds) {
+            if (this.cache.has(`${workspaceId}/${documentId}`))
+                continue;
+            const document = await this.read(workspaceId, documentId);
+            if (document)
+                this.cache.set(`${workspaceId}/${documentId}`, new Map(document.records.map(record => [record.chunk, record])));
+        }
+        const available = keys.filter(([, documentId]) => this.cache.get(`${workspaceId}/${documentId}`)?.size);
+        if (!available.length)
+            return null;
+        const vector = await this.embed([query]);
+        if (!vector?.[0])
+            return null;
+        const scores = new Map();
+        for (const [key, documentId] of available) {
+            const chunk = Number(key.split("#")[1]);
+            const record = this.cache.get(`${workspaceId}/${documentId}`)?.get(chunk);
+            if (!record)
+                continue;
+            const similarity = cosine(vector[0], record.values);
+            if (similarity >= MIN_COSINE)
+                scores.set(key, similarity);
+        }
+        return scores;
+    }
+}