@evandu/super-pi 0.1.84 → 0.1.85

package/dist/server/knowledge.d.ts

@@ -0,0 +1,263 @@
+import { VectorIndex } from "./knowledge-vectors.js";
+import type { Store } from "./store.js";
+import type { Conversation } from "./types.js";
+/** One document as it lives on disk: the ORIGINAL text, never the chunks. */
+export type KnowledgeDocument = {
+    id: string;
+    workspaceId: string;
+    title: string;
+    /** Where it came from - a workspace-relative path, or `pasted` for hand-entered text. */
+    source: string;
+    format: string;
+    characters: number;
+    createdAt: string;
+    updatedAt: string;
+    /**
+     * Content fingerprint, used to recognise an import the user already made.
+     *
+     * WHY IT IS STORED RATHER THAN COMPUTED ON COMPARISON. Deduplication runs inside `add`, which already
+     * reads every document in the workspace; hashing each one's full text there would be O(corpus) on
+     * every import. Storing it turns the check into one string comparison, and a document written by an
+     * older build simply hashes on read (`digestOf`) instead.
+     */
+    digest: string;
+    text: string;
+};
+/** A retrievable slice. `heading` is the nearest preceding markdown heading, so a hit can be cited. */
+export type KnowledgeChunk = {
+    index: number;
+    heading: string;
+    text: string;
+};
+export type KnowledgeHit = {
+    documentId: string;
+    title: string;
+    source: string;
+    heading: string;
+    chunk: number;
+    score: number;
+    text: string;
+};
+/** 1 MB. Big enough for a design doc or a spec, small enough that chunking stays instant. */
+export declare const MAX_DOCUMENT_BYTES: number;
+/** A pasted document rides inside a JSON body, so the request cap leaves room for escaping overhead. */
+export declare const MAX_KNOWLEDGE_REQUEST_BYTES: number;
+/** Per workspace, across every document. A knowledge base is a curated asset, not a file server. */
+export declare const MAX_WORKSPACE_BYTES: number;
+export declare const DEFAULT_RESULT_LIMIT = 6;
+export declare const MAX_RESULT_LIMIT = 20;
+/** What one search may be asked for. `rerank` and `conversation` exist only for the agent's own calls. */
+export type KnowledgeSearch = {
+    limit?: number;
+    /** Re-order with the session's model. Off for the settings preview, which must stay instant. */
+    rerank?: boolean;
+    /** Required by `rerank`: it answers with this conversation's model and workspace. */
+    conversation?: Conversation;
+    signal?: AbortSignal;
+};
+/**
+ * The slice of the runtime the reranker needs, declared structurally.
+ *
+ * Declared here rather than imported so this file does not depend on the spawn machinery that depends
+ * on it - `runtime.ts` imports `KNOWLEDGE_TOOL_NAME` from here, and a type-only back edge into that
+ * module would make the cycle visible in every import graph.
+ */
+export type KnowledgeRuntime = {
+    answer(input: {
+        conversation: Conversation;
+        prompt: string;
+        cwd: string;
+        signal: AbortSignal;
+        knowledgeRerank?: boolean;
+    }): AsyncIterable<string>;
+};
+/**
+ * The first integer array in the model's answer, or null.
+ *
+ * WHY TOLERANT PARSING. The prompt asks for a bare JSON array, and a model that mostly complies will
+ * still sometimes wrap it in a sentence or a code fence. Recovering the array from that is worth doing;
+ * guessing when there is no array at all is not, so anything unparseable is null and the caller keeps
+ * the lexical order.
+ */
+export declare function parseRanking(output: string): number[] | null;
+/** Duplicated as a literal in `agents/super-pi-knowledge/index.ts`, the way `TASK_TOOL_NAME` is. */
+export declare const KNOWLEDGE_TOOL_NAME = "knowledge_search";
+/**
+ * Index terms for one string: CJK bigrams plus Latin words.
+ *
+ * WHY BIGRAMS AND NOT WORDS. Chinese has no spaces, so the word-list tokenizer the rest of this
+ * server uses (`ActiveMemory.search`) returns one enormous "word" for a whole sentence and matches
+ * nothing usefully. Character bigrams need no dictionary and no dependency, and they carry enough
+ * precision that 知识库 and 记忆库 share 识库 but not the whole run - which is what makes a two-term
+ * overlap meaningful rather than accidental.
+ */
+export declare function tokenize(text: string): string[];
+/**
+ * The longer-range channel: character TRIGRAMS, for the passage that repeats a phrase verbatim.
+ *
+ * WHY A SECOND GRANULARITY AND NOT JUST BIGRAMS. Bigrams answer "do these share any two-character
+ * run", which is high recall but low precision - two unrelated Chinese sentences collide constantly on
+ * a shared pair. Trigrams only fire on a real three-character overlap, so when the user's question
+ * quotes the document's own wording this channel separates it sharply from the rest of the corpus.
+ * It is scored on its own and fused with the bigram channel rather than merged into it, because the
+ * two answer different questions and a term that is strong for one is usually weak for the other.
+ */
+export declare function tokenizePhrases(text: string): string[];
+/** The identity of a document's CONTENT, independent of its name, its path and when it arrived. */
+export declare function digestOf(text: string): string;
+/**
+ * Splits a document into overlapping chunks, remembering the heading each one sits under.
+ *
+ * WHY OVERLAP. A passage that spans a boundary is the passage most likely to be the answer - the
+ * sentence that defines a term usually starts in one chunk and ends in the next - and a retrieval
+ * system that cut there would return a fragment ending mid-thought. The 150-character tail is
+ * repeated at the head of the next chunk to keep that sentence whole in at least one hit.
+ */
+export declare function chunkDocument(text: string): KnowledgeChunk[];
+export declare class KnowledgeBase {
+    readonly directory: string;
+    private store;
+    private runtime?;
+    private vectors?;
+    private capabilities;
+    private indexes;
+    private serial;
+    constructor(directory: string, store: Store, runtime?: KnowledgeRuntime | undefined, vectors?: VectorIndex | undefined);
+    /** `SUPER_PI_KNOWLEDGE_ENABLED=0` turns the whole feature off, the same way memory turns off. */
+    enabled(): boolean;
+    /** The opaque workspace identity on disk, encoded. Paths never come from the model or the request. */
+    private folder;
+    private file;
+    private requireWorkspace;
+    private locked;
+    /** Metadata only - the caller listing documents must never receive the full text of a workspace. */
+    private summarise;
+    private readAll;
+    private save;
+    private index;
+    /**
+     * Loads (or rebuilds) one workspace's index. Cache miss is the only thing that reads the disk.
+     *
+     * WHY REBUILT PER WORKSPACE AND NOT ONE GLOBAL INDEX. Isolation is a security property here, not a
+     * performance one: a hit may only ever come from the workspace the turn's capability names. A
+     * single shared index would need every search to filter by workspace after scoring, which is one
+     * forgotten filter away from a cross-tenant leak. Separate caches cannot express that mistake.
+     *
+     * WHY THERE IS NO ON-DISK INDEX. Measured, not assumed: rebuilding the index of a 16 MB workspace -
+     * the feature's own capacity cap - costs about 180 ms, once per process. A serialised index would be
+     * larger than the corpus it describes and JSON.parse of it is not reliably cheaper than the
+     * tokenisation, so it would trade a correctness-neutral optimisation for a second on-disk artefact
+     * that can drift from the documents.
+     */
+    private load;
+    private invalidate;
+    list(workspaceId: string): Promise<Array<Omit<KnowledgeDocument, "text">>>;
+    /**
+     * Whether this workspace has any document at all, which is what decides whether the turn mounts the
+     * extension.
+     *
+     * WHY A DIRECTORY LISTING AND NOT `list()`. This runs once per turn, before every spawn. `list()`
+     * would read and parse every document's full text to answer a yes/no question, so a workspace with
+     * a dozen real documents would pay for parsing all of them on every message - including the
+     * catalogue fetch the extension makes the moment it loads. `readdir` answers the same question for
+     * the price of one directory entry, and a name is all this needs.
+     */
+    hasDocuments(workspaceId: string): Promise<boolean>;
+    /**
+     * Adds a document, either from a workspace file or from pasted text.
+     *
+     * WHY THE FILE IS READ HERE AND NOT UPLOADED. A knowledge base is made of documents the person
+     * ALREADY has in their workspace - re-uploading a copy of the file would give the agent a second,
+     * newer version that diverges from the one the workspace serves. The path therefore goes through
+     * the same `resolveInsideRoot` guard the file editor uses, so symlink and `..` escapes are refused
+     * by construction instead of by a check someone has to remember.
+     */
+    add(workspaceId: string, body: {
+        path?: string;
+        title?: string;
+        text?: string;
+    }): Promise<Omit<KnowledgeDocument, "text">>;
+    remove(workspaceId: string, documentId: string): Promise<void>;
+    /**
+     * Scores this workspace's chunks on four independent lexical channels and fuses them.
+     *
+     * WHY LEXICAL AND NOT ONLY EMBEDDINGS. See docs/KNOWLEDGE_BASE.md: the vector channel is fused in
+     * here rather than replacing this one, because each is wrong in a way the other catches. BM25
+     * misses a paraphrase completely; a vector search will happily return the nearest thing at a
+     * confident-looking 0.58 when the answer is not in the knowledge base at all. Summing a normalised
+     * lexical score with a normalised cosine score keeps both signals - and keeps "no hit" reachable.
+     *
+     * WHY NORMALISE PER CHANNEL AND SUM, RATHER THAN RRF. Reciprocal rank fusion is the usual way to
+     * combine ranked lists, and it would be a regression here: it discards score magnitude, so a
+     * passage that matches decisively and one that matches barely would arrive at the same rank and
+     * finish within a couple of percent of each other. The gap between a clear winner and a near miss is
+     * exactly what BM25 was measuring. Each channel is therefore divided by its own best score for this
+     * query, which preserves the shape of that channel's confidence while making the weights comparable.
+     */
+    search(workspaceId: string, query: string, options?: KnowledgeSearch): Promise<KnowledgeHit[]>;
+    /**
+     * Re-orders the candidate pool with the session's own model.
+     *
+     * WHY THIS EXISTS AND WHAT IT IS WORTH. Lexical retrieval ranks on shared characters, so a passage
+     * that answers in different words sits below passages that merely repeat the question's wording -
+     * often below the fold, where the character budget has already cut it off. Reading the candidates
+     * is something the model can already do, so it is asked to pick.
+     *
+     * ITS CEILING, STATED PLAINLY. Reranking can only reorder what the first stage returned. A query
+     * with NO lexical overlap at all never reaches this function, which is why the vector channel and
+     * not this one is what closes the paraphrase gap - see docs/KNOWLEDGE_BASE.md.
+     *
+     * WHY IT FAILS OPEN. Every outcome that is not a clean parse returns the lexical order. A reranker
+     * that made things worse, hung, or answered with prose would degrade the feature invisibly, and a
+     * retrieval layer that is allowed to get worse by being clever is worse than one that is not.
+     */
+    private rerank;
+    /**
+     * Mints a read-only token for one turn. The agent can retrieve, never write: everything a person
+     * puts in a knowledge base is theirs to curate from the settings window.
+     */
+    issue(conversationId: string): string;
+    release(token: string): void;
+    private identity;
+    /** The agent-facing endpoint: a title catalogue and a search. Nothing that writes. */
+    tool(token: string, body: {
+        kind?: string;
+        query?: string;
+        limit?: number;
+    }, signal?: AbortSignal): Promise<{
+        documents: {
+            title: string;
+            source: string;
+            characters: number;
+        }[];
+        hits?: undefined;
+        reranked?: undefined;
+    } | {
+        documents?: undefined;
+        hits: KnowledgeHit[];
+        reranked: boolean;
+    }>;
+    /** The browser-facing endpoint, behind the ordinary access token and same-origin check. */
+    manage(workspaceId: string, body: {
+        action?: string;
+        path?: string;
+        title?: string;
+        text?: string;
+        documentId?: string;
+        query?: string;
+        limit?: number;
+    }): Promise<{
+        hits: KnowledgeHit[];
+        documents: Omit<KnowledgeDocument, "text">[];
+        semantic: import("./knowledge-vectors.js").VectorStatus | null;
+    }>;
+    /**
+     * What the panel shows, including whether semantic retrieval is actually working.
+     *
+     * WHY THE STATE IS WORTH SHOWING. The vector half depends on an optional package and a download from
+     * a host the machine may not reach, and both fail quietly by design. Without this on screen, "the
+     * feature is off" is indistinguishable from "the feature is broken here" - which is the difference
+     * between not wanting a feature and not being able to have it.
+     */
+    private overview;
+}