@cerebriline/core 4.100.240 → 4.100.241

package/dist/services/retrieval/firecrawl.d.ts

@@ -0,0 +1,77 @@
+/**
+ * A Firecrawl endpoint, as the Library's scraper uses it: one page, a crawl
+ * from a page outwards, the links of a site, and a web search.
+ *
+ * Pages are asked for as HTML and turned into markdown here. Firecrawl's own
+ * markdown loses fenced code blocks, which is most of what a book made from
+ * documentation is for.
+ */
+export interface ScrapeEndpoint {
+    /** The endpoint's address: `http://host:3002`. */
+    baseUrl: string;
+    apiKey?: string;
+}
+export interface ScrapedPage {
+    url: string;
+    title?: string;
+    description?: string;
+    language?: string;
+    markdown: string;
+    /** A hash of the markdown: whether the page changed since it was last read. */
+    sha256: string;
+    /** The links on the page, when they were asked for. */
+    links?: string[];
+}
+export interface ScrapeFailure {
+    url: string;
+    reason: string;
+}
+export interface CrawlOptions {
+    /** Pages read, at most. */
+    limit: number;
+    /** How many links deep from the starting page. */
+    depth: number;
+    /** Only paths matching these (regular expressions), when given. */
+    includePaths?: readonly string[];
+    excludePaths?: readonly string[];
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+    /** How long to wait for the crawl. @default 20 minutes */
+    timeoutMs?: number;
+    onProgress?: (done: number, total: number) => void;
+}
+export interface CrawlResult {
+    pages: ScrapedPage[];
+    failed: ScrapeFailure[];
+    /** Set when the crawl was still running at the time limit. */
+    unfinished?: boolean;
+}
+export interface WebSearchHit {
+    url: string;
+    title?: string;
+    description?: string;
+}
+interface RequestOptions {
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+}
+export declare class ScrapeError extends Error {
+}
+/** A page's HTML as markdown, with its code blocks fenced. */
+export declare function htmlToMarkdown(html: string): Promise<string>;
+/** Read one page. */
+export declare function scrapePage(endpoint: ScrapeEndpoint, url: string, options?: RequestOptions & {
+    links?: boolean;
+}): Promise<ScrapedPage>;
+/** Read a page and the pages it links to, as deep and as many as `options` allow. */
+export declare function crawlSite(endpoint: ScrapeEndpoint, url: string, options: CrawlOptions): Promise<CrawlResult>;
+/** The pages of a site, by their links, without reading them. */
+export declare function mapSite(endpoint: ScrapeEndpoint, url: string, options?: RequestOptions & {
+    limit?: number;
+    search?: string;
+}): Promise<string[]>;
+/** Search the web. Works when the endpoint has a search provider behind it. */
+export declare function searchWeb(endpoint: ScrapeEndpoint, query: string, options?: RequestOptions & {
+    limit?: number;
+}): Promise<WebSearchHit[]>;
+export {};

package/dist/services/retrieval/lancedb-runtime.d.ts

@@ -0,0 +1,84 @@
+/**
+ * LanceDB, fetched when it is first needed.
+ *
+ * The vector index of the Library is LanceDB, whose native library is 200 to
+ * 390 MB depending on the platform. That cannot ship in the extension, so it
+ * is downloaded once into the data folder, the way OCR languages are: the
+ * JavaScript packages and the one native package for this machine, straight
+ * from the npm registry, each checked against the integrity pinned in
+ * `lancedb-runtime.manifest.ts` before anything is used.
+ *
+ * Keyword search does not depend on any of this.
+ */
+import { type LanceDbRuntimeManifest } from "./lancedb-runtime.manifest";
+export interface LanceDbInstallProgress {
+    /** The package being fetched, and how many there are. */
+    packageIndex: number;
+    packageCount: number;
+    path: string;
+    /** Bytes of this package received so far, and its size when the server said. */
+    receivedBytes: number;
+    totalBytes?: number;
+}
+export interface LanceDbRuntimeOptions {
+    /** The folder runtimes are kept in, e.g. `<data>/runtimes/lancedb`. */
+    directory: string;
+    manifest?: LanceDbRuntimeManifest;
+    fetch?: typeof fetch;
+    onProgress?: (progress: LanceDbInstallProgress) => void;
+    signal?: AbortSignal;
+    /** For tests: the platform key to install for. */
+    platformKey?: string;
+}
+export interface LanceDbRuntime {
+    /** The `@lancedb/lancedb` module. */
+    lancedb: any;
+    /** The `apache-arrow` module LanceDB was installed with. */
+    arrow: any;
+    version: string;
+    root: string;
+}
+export declare class LanceDbUnavailableError extends Error {
+    constructor(message: string);
+}
+/** `linux-x64-gnu`, `win32-x64`, `darwin-arm64`: the key into the manifest. */
+export declare function currentPlatformKey(): string;
+export declare function lanceDbRuntimeRoot(options: Pick<LanceDbRuntimeOptions, "directory" | "manifest">): string;
+export declare function isLanceDbInstalled(options: Pick<LanceDbRuntimeOptions, "directory" | "manifest">): boolean;
+/** Why LanceDB cannot run here, or undefined when it can. */
+export declare function lanceDbUnsupportedReason(options?: Pick<LanceDbRuntimeOptions, "manifest" | "platformKey">): string | undefined;
+/** How much will be downloaded and unpacked for this platform, when known. */
+export declare function lanceDbInstallBytes(options?: Pick<LanceDbRuntimeOptions, "manifest" | "platformKey">): number | undefined;
+/**
+ * Download and unpack LanceDB for this machine, unless it is already there.
+ * Everything lands in a folder of its own first and is moved into place only
+ * when every package has been verified, so an interrupted or failed install
+ * leaves nothing that looks installed.
+ */
+export declare function installLanceDb(options: LanceDbRuntimeOptions): Promise<string>;
+/** Load an installed LanceDB. Throws `LanceDbUnavailableError` when it is not installed. */
+export declare function loadLanceDb(options: Pick<LanceDbRuntimeOptions, "directory" | "manifest">): LanceDbRuntime;
+/** Where LanceDB stands here: for a settings panel, and for a bug report. */
+export interface LanceDbStatus {
+    /** The version this build installs. */
+    version: string;
+    platform: string;
+    installed: boolean;
+    /** Installed, and the native library loaded in this process. */
+    working: boolean;
+    /** Why it did not load, when it is installed and did not. */
+    error?: string;
+    /** Why it cannot run here at all. */
+    unsupported?: string;
+    /** What a download unpacks to, when known. */
+    installBytes?: number;
+    root: string;
+}
+/**
+ * Whether LanceDB is installed and actually loads. Loading is the part
+ * worth checking: the files being there says nothing about whether this
+ * process, with its Node and its C library, can open the native library.
+ */
+export declare function lanceDbStatus(options: Pick<LanceDbRuntimeOptions, "directory" | "manifest">): LanceDbStatus;
+/** Install if needed, then load. */
+export declare function ensureLanceDb(options: LanceDbRuntimeOptions): Promise<LanceDbRuntime>;

package/dist/services/retrieval/lancedb-runtime.manifest.d.ts

@@ -0,0 +1,17 @@
+export interface LanceDbRuntimePackage {
+    /** Where it is unpacked, relative to the runtime folder. */
+    path: string;
+    tarball: string;
+    /** `sha512-<base64>`, as npm records it. */
+    integrity: string;
+    unpackedBytes?: number;
+}
+export interface LanceDbRuntimeManifest {
+    version: string;
+    arrow: string;
+    /** The JavaScript packages, the same on every platform. */
+    packages: LanceDbRuntimePackage[];
+    /** The native library, by `<platform>-<arch>` with `-gnu`/`-musl` on Linux. */
+    platforms: Record<string, LanceDbRuntimePackage>;
+}
+export declare const LANCEDB_RUNTIME_MANIFEST: LanceDbRuntimeManifest;

package/dist/services/retrieval/library-store.d.ts

@@ -0,0 +1,148 @@
+/**
+ * The Library's text store: collections, their documents, and the chunks
+ * each document was cut into, in one SQLite file with a full-text index.
+ *
+ * This is the part of the Library that needs no model. Keyword search works
+ * from the moment a document is added; vectors, when an embedding model is
+ * set, are kept elsewhere and refer to a chunk by the id it has here.
+ *
+ * The index is FTS5's external-content form, so the text of a library of
+ * books is stored once, in `chunks`, and not a second time in the index.
+ */
+import { type SqliteDb } from "@cerebriline/shared/db";
+import type { TextChunk } from "./chunker";
+export interface LibraryCollection {
+    id: number;
+    name: string;
+    createdAt: string;
+    documents: number;
+    chunks: number;
+}
+export interface LibraryDocumentInput {
+    /** Where it came from: a path or a URL. One document per source in a collection. */
+    source: string;
+    title?: string;
+    /** A hash of the content. A source added again with the same hash is left alone. */
+    contentHash: string;
+    bytes?: number;
+    /** Anything else worth keeping: author, language, pages. */
+    metadata?: Record<string, unknown>;
+}
+export interface LibraryDocument {
+    id: number;
+    collectionId: number;
+    source: string;
+    title?: string;
+    contentHash: string;
+    bytes?: number;
+    chunks: number;
+    addedAt: string;
+    metadata: Record<string, unknown>;
+}
+export interface AddDocumentResult {
+    document: LibraryDocument;
+    /** "unchanged" when the same content was already there and nothing was written. */
+    outcome: "added" | "replaced" | "unchanged";
+    /** The ids given to the chunks, in order. Empty for "unchanged". */
+    chunkIds: number[];
+    /**
+     * For "replaced": the id the document had before. Its chunks are gone, and
+     * whatever else was kept under that id (its vectors) is the caller's to remove.
+     */
+    replacedDocumentId?: number;
+}
+export interface KeywordSearchOptions {
+    /** Collections to search. All of them when left out. */
+    collectionIds?: readonly number[];
+    /** Hits returned. Default 10. */
+    limit?: number;
+    /**
+     * Also match the document's file name, title and section headers, at this
+     * weight against the text's 1. 0 searches the text alone. Default 0.5.
+     */
+    contextWeight?: number;
+}
+export interface LibraryHit {
+    chunkId: number;
+    documentId: number;
+    collectionId: number;
+    /** Higher is better. BM25 for keyword hits; not comparable across queries. */
+    score: number;
+    text: string;
+    start: number;
+    end: number;
+    headings: string[];
+    source: string;
+    title?: string;
+}
+/** The words of a query, as the index's tokenizer would see them. */
+export declare function queryTerms(query: string): string[];
+/** What the index also matches on for a chunk: file name, title, headers. */
+export declare function chunkContext(document: Pick<LibraryDocumentInput, "source" | "title">, headings: readonly string[]): string;
+export declare class LibraryStore {
+    private readonly db;
+    constructor(filePath: string);
+    close(): void;
+    /** The database itself, for the catalogue kept in the same file. */
+    get database(): SqliteDb;
+    transaction<T>(run: () => T): T;
+    /** The collection of that name, created if it is not there. */
+    ensureCollection(name: string): LibraryCollection;
+    /** Give a collection another name. The documents and vectors stay as they are. */
+    renameCollection(collectionId: number, name: string): void;
+    listCollections(): LibraryCollection[];
+    /** Removes the collection with its documents and chunks. */
+    deleteCollection(collectionId: number): void;
+    private toDocument;
+    listDocuments(collectionId: number): LibraryDocument[];
+    findDocument(collectionId: number, source: string): LibraryDocument | undefined;
+    /**
+     * Adds a document with its chunks, in one transaction. The same source
+     * with the same content is left as it is; with different content, the old
+     * chunks are replaced.
+     */
+    addDocument(collectionId: number, input: LibraryDocumentInput, chunks: readonly TextChunk[]): AddDocumentResult;
+    deleteDocument(documentId: number): void;
+    getDocument(documentId: number): LibraryDocument | undefined;
+    /** A document's chunks, in order: what is embedded. */
+    documentChunks(documentId: number): LibraryHit[];
+    /** Documents with chunks and no vectors yet for this model, oldest first. */
+    /**
+     * The documents with no vectors for this model at the size it returns
+     * now. The size is the one last seen (`noteDimension`) unless given; a
+     * model never seen has no size to hold a document to.
+     */
+    documentsWithoutEmbedding(model: string, collectionIds?: readonly number[], dimension?: number | undefined): LibraryDocument[];
+    /** Record the vector size a model returned. True when it differs from the last one seen. */
+    noteDimension(model: string, dimension: number): boolean;
+    knownDimension(model: string): number | undefined;
+    /** Every model and size documents are recorded as embedded with, and how many. */
+    embeddingSets(): {
+        model: string;
+        dimension: number;
+        documents: number;
+    }[];
+    /** Forget that documents were embedded with this model at this size: their vectors are gone. */
+    forgetEmbeddings(model: string, dimension: number): void;
+    markEmbedded(documentId: number, model: string, dimension: number): void;
+    /** Totals, and how many documents have vectors for this model. */
+    counts(model?: string, dimension?: number | undefined): {
+        collections: number;
+        documents: number;
+        chunks: number;
+        embeddedDocuments: number;
+    };
+    private toHit;
+    /** Chunks by id, in the order asked for. Unknown ids are left out. */
+    getChunks(chunkIds: readonly number[]): LibraryHit[];
+    /**
+     * The query's terms worth searching for: those that are not in a large
+     * share of all chunks. If that leaves none, all of them are kept.
+     */
+    private selectiveTerms;
+    /**
+     * Keyword search, ranked by BM25. A chunk matches when it holds any of
+     * the query's terms; the more of them and the rarer they are, the higher.
+     */
+    searchKeywords(query: string, options?: KeywordSearchOptions): LibraryHit[];
+}

package/dist/services/retrieval/library.d.ts

@@ -0,0 +1,139 @@
+/**
+ * The Library: documents kept for retrieval.
+ *
+ * It ties the pieces together. Text goes in as chunks in SQLite, where it is
+ * searchable by keyword at once; vectors are a second pass that needs an
+ * embedding model and LanceDB, and can be run later, resumed, or never.
+ */
+import { type LibrarySettings } from "@cerebriline/shared";
+import { Catalogue } from "./catalogue";
+import { type RetrievalEndpoint } from "./embedding-client";
+import { type LanceDbInstallProgress } from "./lancedb-runtime";
+import { type AddDocumentResult, type LibraryDocument, LibraryStore } from "./library-store";
+import { type RetrieveResult } from "./retrieve";
+import { VectorIndex } from "./vector-index";
+export declare function resolveLibraryDirectory(): string;
+export declare function resolveLanceDbRuntimeDirectory(): string;
+export interface LibraryOptions {
+    /** Where the Library is kept. @default <data dir>/library */
+    directory?: string;
+    /** Where LanceDB is installed. @default <data dir>/runtimes/lancedb */
+    runtimeDirectory?: string;
+    fetch?: typeof fetch;
+}
+export interface LibraryDocumentText {
+    source: string;
+    title?: string;
+    /** The document as text or markdown. */
+    text: string;
+    bytes?: number;
+    metadata?: Record<string, unknown>;
+}
+export interface EmbedProgress {
+    document: LibraryDocument;
+    documentIndex: number;
+    documentCount: number;
+    chunksDone: number;
+}
+export interface EmbedPendingOptions {
+    embedding: RetrievalEndpoint;
+    settings?: LibrarySettings;
+    collectionIds?: readonly number[];
+    /**
+     * Download LanceDB when it is not installed. Off, a Library with no
+     * LanceDB embeds nothing and says why.
+     */
+    install?: boolean;
+    onInstallProgress?: (progress: LanceDbInstallProgress) => void;
+    onProgress?: (progress: EmbedProgress) => void;
+    /**
+     * Ask the model for one vector first, to learn the size it returns now.
+     * For a run the user asked for: without it, a model that changed size
+     * since everything was embedded looks finished until the next search.
+     */
+    probe?: boolean;
+    signal?: AbortSignal;
+}
+export interface EmbedPendingResult {
+    documents: number;
+    chunks: number;
+    dimension?: number;
+    /** Why nothing was embedded, when that is so. */
+    skipped?: string;
+}
+export interface LibraryVectorSet {
+    /** What `deleteVectorSet` takes. */
+    table: string;
+    model: string;
+    dimension: number;
+    vectors: number;
+    /** Documents recorded as embedded with it. */
+    documents: number;
+    bytes: number;
+    /** The set the embedding model now set writes to and searches. */
+    current: boolean;
+}
+export interface LibrarySearchOptions {
+    settings?: LibrarySettings;
+    collectionIds?: readonly number[];
+    embedding?: RetrievalEndpoint;
+    reranker?: RetrievalEndpoint;
+    signal?: AbortSignal;
+}
+export declare class Library {
+    readonly store: LibraryStore;
+    /** Where this Library is kept. */
+    readonly directory: string;
+    private readonly runtimeDirectory;
+    private readonly fetch?;
+    private index?;
+    private shelves?;
+    constructor(options?: LibraryOptions);
+    /**
+     * The sections, shelves and books. Made on first use: Memory keeps its
+     * notes in a store of the same kind and has no shelves.
+     */
+    get catalogue(): Catalogue;
+    /** Whether vectors can be used here now, and if not, why. */
+    vectorState(): {
+        installed: boolean;
+        unsupported?: string;
+    };
+    /** The vector index, when LanceDB is installed (or `install` asks for it). */
+    vectors(options?: {
+        install?: boolean;
+        onInstallProgress?: (progress: LanceDbInstallProgress) => void;
+        signal?: AbortSignal;
+    }): Promise<VectorIndex | undefined>;
+    /**
+     * Add a document's text to a collection. It is searchable by keyword as
+     * soon as this returns; `embedPending` gives it vectors.
+     */
+    addDocument(collection: string, document: LibraryDocumentText, settings?: LibrarySettings): Promise<AddDocumentResult>;
+    private dropVectors;
+    removeDocument(documentId: number): Promise<void>;
+    removeCollection(collectionId: number): Promise<void>;
+    /**
+     * Give vectors to every document that has none for this embedding model.
+     * A document is marked done only when all of its vectors are written, so
+     * a run that is stopped or fails is picked up where it left off.
+     */
+    embedPending(options: EmbedPendingOptions): Promise<EmbedPendingResult>;
+    /**
+     * The sets of vectors on disk: one per embedding model and vector size.
+     * A change of model starts a new set and keeps the old one, so going back
+     * costs nothing -- and so they add up until one is deleted.
+     */
+    vectorSets(current?: string): Promise<LibraryVectorSet[]>;
+    /** Delete one set of vectors, and forget that its documents were embedded with it. */
+    deleteVectorSet(table: string): Promise<boolean>;
+    search(query: string, options?: LibrarySearchOptions): Promise<RetrieveResult>;
+    close(): Promise<void>;
+}
+/**
+ * Closes every Library this process opened. For shutdown: the SQLite file is
+ * checkpointed and released, and the next `sharedLibrary()` opens it afresh.
+ */
+export declare function closeSharedLibraries(): Promise<void>;
+/** The one Library of a folder in this process: SQLite and LanceDB are opened once. */
+export declare function sharedLibrary(options?: LibraryOptions): Library;

package/dist/services/retrieval/memory-recall.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Automatic recall: Memory searched for every message the user sends, and
+ * what is found put beside the message.
+ *
+ * The model has `recall`, and is told to call it at the start of a task. A
+ * model that does not ask does not get its notes, and small models mostly do
+ * not ask. So the host asks for it.
+ *
+ * The order of work is the one claude-hooks settled on after measuring it
+ * (https://github.com/mann1x/claude-hooks, docs/hyde.md):
+ *
+ * 1. Search with the user's own words. Always.
+ * 2. HyDE (Gao et al., "Precise Zero-Shot Dense Retrieval without Relevance
+ *    Labels", 2022), when it is on and step 1 found something: a second model
+ *    writes the note that would answer the message, given the notes just
+ *    found, and Memory is searched again with that. Grounded, because a model
+ *    asked to imagine a note about a project's own jargon invents one about
+ *    something else. Skipped when Memory holds no note in reach, for the
+ *    same reason: there is nothing to ground on, and nothing to find.
+ * 3. The two results are fused, the first search weighing as much as the
+ *    second.
+ *
+ * One thing is added to that. A search always returns its best matches,
+ * however poor: by keyword, "the" and "project" match every note. Asked for
+ * by the model that is fine, it asked. Attached unasked to every message it
+ * is noise, so a note is attached only when something says it is about the
+ * message: the reranker passed it, or it is close by meaning, or it shares
+ * two meaningful words with what was searched for.
+ *
+ * Nothing here can fail a turn. A search that throws, an expander that times
+ * out, a server that is down: the message goes to the model without notes.
+ */
+import { type MemoryToolsConfig } from "../../extensions/tools/memory-tools";
+import { type Memory, type RecalledMemory } from "./memory";
+/** A message shorter than this is "yes", "continue", "go on": nothing to search for. */
+export declare const MEMORY_RECALL_MIN_PROMPT_CHARS = 12;
+/** A pasted log is not a question. Both ends are kept: the ask is at one of them. */
+export declare const MEMORY_RECALL_MAX_QUERY_CHARS = 2000;
+/** Notes given to the expander to ground on, and how much of them. */
+export declare const HYDE_GROUND_NOTES = 3;
+export declare const HYDE_GROUND_MAX_CHARS = 1500;
+/** The expander's time. Past it the first search stands alone. */
+export declare const HYDE_TIMEOUT_MS = 20000;
+/**
+ * How close by meaning a note has to be to be attached unasked. A starting
+ * point: it depends on the embedding model and is to be tuned.
+ */
+export declare const AUTO_RECALL_MIN_SIMILARITY = 0.5;
+/** Meaningful words a note has to share with the search to be attached unasked. */
+export declare const AUTO_RECALL_MIN_SHARED_WORDS = 2;
+export declare const HYDE_SYSTEM_PROMPT = "You are a memory recall assistant. You are given a user's message and a few related notes retrieved from a knowledge base. Using ONLY facts consistent with those notes, write the short note that would answer the message, as if it were itself a stored note. Prefer the terms and specifics of the notes given. Do not invent facts beyond them. Two or three sentences. State facts; do not explain, and do not address the user.";
+export declare function buildHydeUserPrompt(question: string, notes: string[]): string;
+/** Writes the hypothetical note. Undefined or empty: search without it. */
+export type MemoryQueryExpander = (input: {
+    system: string;
+    prompt: string;
+    signal: AbortSignal;
+}) => Promise<string | undefined>;
+export interface MemoryRecallInput {
+    /** The user's message, as typed. */
+    prompt: string;
+    /** The workspace, which decides the memories that are searched. */
+    cwd: string;
+    /** Notes already given to a session are not given to it again. */
+    sessionId?: string;
+}
+export interface MemoryRecallResult {
+    /** The block to put after the user's message. Absent: nothing found. */
+    context?: string;
+    /** The notes in it. */
+    ids: string[];
+    /** Whether the expander's note was searched with. */
+    expanded: boolean;
+    /** One line for the user: what was recalled. */
+    summary?: string;
+}
+export type MemoryRecaller = (input: MemoryRecallInput) => Promise<MemoryRecallResult | undefined>;
+export interface CreateMemoryRecallerOptions {
+    /** Read on every message. Undefined, or Memory off: no recall. */
+    getConfig: () => MemoryToolsConfig | undefined;
+    /** Read on every message, so a profile chosen mid-session is used. */
+    getExpander?: () => MemoryQueryExpander | undefined;
+    /** @default the shared Memory of the data folder */
+    memory?: Memory;
+    log?: (message: string) => void;
+    hydeTimeoutMs?: number;
+}
+/** The meaningful words of a text, stemmed. */
+export declare function contentWords(text: string): Set<string>;
+/** Whether a note found for a search is about it, and not merely the best of a poor lot. */
+export declare function isAboutTheSearch(search: string, item: RecalledMemory): boolean;
+/** Bound a query, keeping its head and its tail. */
+export declare function clampRecallQuery(query: string, maxChars?: number): string;
+/** Reciprocal-rank fusion of two orderings of notes, best first. */
+export declare function fuseRecalls(first: RecalledMemory[], second: RecalledMemory[], limit: number): RecalledMemory[];
+export declare function formatRecalledMemory(items: RecalledMemory[]): string;
+export declare function createMemoryRecaller(options: CreateMemoryRecallerOptions): MemoryRecaller;