@cerebriline/core 4.100.240 → 4.100.241

package/dist/services/global-settings.d.ts

@@ -34,6 +34,9 @@ export declare const GlobalSettingsSchema: z.ZodPipe<z.ZodObject<{
     }, z.core.$strip>>>;
     disabledPlugins: z.ZodOptional<z.ZodPipe<z.ZodPreprocess<z.ZodOptional<z.ZodArray<z.ZodString>>>, z.ZodTransform<string[] | undefined, string[] | undefined>>>;
     disabledAgentPlugins: z.ZodOptional<z.ZodPipe<z.ZodPreprocess<z.ZodOptional<z.ZodArray<z.ZodString>>>, z.ZodTransform<string[] | undefined, string[] | undefined>>>;
+    bundledSkills: z.ZodCatch<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+        enabled: z.ZodBoolean;
+    }, z.core.$strip>>>>;
 }, z.core.$strip>, z.ZodTransform<{
     telemetryOptOut: boolean;
     autoUpdateEnabled: boolean;
@@ -46,6 +49,9 @@ export declare const GlobalSettingsSchema: z.ZodPipe<z.ZodObject<{
     tools?: ModelToolSettings;
     disabledPlugins?: string[];
     disabledAgentPlugins?: string[];
+    bundledSkills?: Record<string, {
+        enabled: boolean;
+    }>;
 }, {
     telemetryOptOut: boolean;
     autoUpdateEnabled: boolean;
@@ -60,6 +66,9 @@ export declare const GlobalSettingsSchema: z.ZodPipe<z.ZodObject<{
     }>> | undefined;
     disabledPlugins?: string[] | undefined;
     disabledAgentPlugins?: string[] | undefined;
+    bundledSkills?: Record<string, {
+        enabled: boolean;
+    }> | undefined;
 }>>;
 export type GlobalSettings = z.infer<typeof GlobalSettingsSchema>;
 export interface WriteGlobalSettingsOptions {

package/dist/services/llms/provider-settings.d.ts

@@ -15,25 +15,25 @@ export declare const normalizeProviderId: typeof Llms.normalizeProviderId;
 export type ProviderDefaultsConfig = ProviderDefaults;
 export declare const ProviderIdSchema: z.ZodString;
 export declare const ProviderProtocolSchema: z.ZodEnum<{
-    "openai-responses": "openai-responses";
     anthropic: "anthropic";
     gemini: "gemini";
+    "openai-chat": "openai-chat";
+    "openai-responses": "openai-responses";
     "openai-r1": "openai-r1";
     "ai-sdk": "ai-sdk";
-    "openai-chat": "openai-chat";
 }>;
 export declare const ProviderClientSchema: z.ZodEnum<{
     custom: "custom";
     fetch: "fetch";
     anthropic: "anthropic";
-    "openai-compatible": "openai-compatible";
     gemini: "gemini";
-    vertex: "vertex";
-    bedrock: "bedrock";
-    openai: "openai";
     "openai-r1": "openai-r1";
     "ai-sdk": "ai-sdk";
     "ai-sdk-community": "ai-sdk-community";
+    openai: "openai";
+    "openai-compatible": "openai-compatible";
+    bedrock: "bedrock";
+    vertex: "vertex";
 }>;
 export declare const AuthSettingsSchema: z.ZodObject<{
     apiKey: z.ZodOptional<z.ZodString>;
@@ -162,9 +162,9 @@ export declare const AwsSettingsSchema: z.ZodObject<{
     region: z.ZodOptional<z.ZodString>;
     profile: z.ZodOptional<z.ZodString>;
     authentication: z.ZodOptional<z.ZodEnum<{
-        "api-key": "api-key";
         profile: "profile";
         iam: "iam";
+        "api-key": "api-key";
         apikey: "apikey";
     }>>;
     usePromptCache: z.ZodOptional<z.ZodBoolean>;
@@ -231,25 +231,25 @@ export declare const ProviderSettingsSchema: z.ZodObject<{
     }, z.core.$strip>>;
     model: z.ZodOptional<z.ZodString>;
     protocol: z.ZodOptional<z.ZodEnum<{
-        "openai-responses": "openai-responses";
         anthropic: "anthropic";
         gemini: "gemini";
+        "openai-chat": "openai-chat";
+        "openai-responses": "openai-responses";
         "openai-r1": "openai-r1";
         "ai-sdk": "ai-sdk";
-        "openai-chat": "openai-chat";
     }>>;
     client: z.ZodOptional<z.ZodEnum<{
         custom: "custom";
         fetch: "fetch";
         anthropic: "anthropic";
-        "openai-compatible": "openai-compatible";
         gemini: "gemini";
-        vertex: "vertex";
-        bedrock: "bedrock";
-        openai: "openai";
         "openai-r1": "openai-r1";
         "ai-sdk": "ai-sdk";
         "ai-sdk-community": "ai-sdk-community";
+        openai: "openai";
+        "openai-compatible": "openai-compatible";
+        bedrock: "bedrock";
+        vertex: "vertex";
     }>>;
     routingProviderId: z.ZodOptional<z.ZodString>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -341,9 +341,9 @@ export declare const ProviderSettingsSchema: z.ZodObject<{
         region: z.ZodOptional<z.ZodString>;
         profile: z.ZodOptional<z.ZodString>;
         authentication: z.ZodOptional<z.ZodEnum<{
-            "api-key": "api-key";
             profile: "profile";
             iam: "iam";
+            "api-key": "api-key";
             apikey: "apikey";
         }>>;
         usePromptCache: z.ZodOptional<z.ZodBoolean>;
@@ -389,8 +389,8 @@ export declare const ProviderSettingsSchema: z.ZodObject<{
         vision: "vision";
         reasoning: "reasoning";
         tools: "tools";
-        streaming: "streaming";
         "prompt-cache": "prompt-cache";
+        streaming: "streaming";
         "computer-use": "computer-use";
         oauth: "oauth";
         popular: "popular";

package/dist/services/retrieval/catalogue.d.ts

@@ -0,0 +1,370 @@
+/**
+ * The Library's catalogue: sections, the shelves in them, and the books on
+ * the shelves.
+ *
+ * A book is a folder (`books/<uid>/`) and a collection in the text store. The
+ * folder is the book itself: the files it was made from, named by their
+ * sha256, the text read out of each, its pictures, and `book.json` saying
+ * what all of it is. The database is the index of those folders, so a folder
+ * can be carried to another Library and read back in, which is what export
+ * and import do.
+ *
+ * Nothing the model does here is final. A deleted book, or a source taken
+ * out of one, goes to the trash and stays there for thirty days.
+ */
+import { type LibrarySettings } from "@cerebriline/shared";
+import type { Library } from "./library";
+export declare const BOOK_FORMAT = "cerebriline-book";
+export declare const LIBRARY_EXPORT_FORMAT = "cerebriline-library";
+/** How long the trash keeps what was deleted. */
+export declare const TRASH_DAYS = 30;
+/** Where the collections of a Library from before it had shelves are put. */
+export declare const LEGACY_SECTION = "General";
+export interface LibraryShelf {
+    id: number;
+    sectionId: number;
+    name: string;
+    description: string;
+    books: number;
+    passages: number;
+}
+export interface LibrarySection {
+    id: number;
+    name: string;
+    description: string;
+    shelves: LibraryShelf[];
+}
+/** Where a scraped book came from, kept so it can be brought up to date. */
+export interface BookWebOrigin {
+    /** What was asked for, when the links were found by searching. */
+    query?: string;
+    /** The links given or found: where a later check starts again. */
+    links: string[];
+    crawl?: {
+        limit?: number;
+        depth?: number;
+    };
+    checkedAt?: string;
+}
+export interface BookMetadata {
+    authors?: string[];
+    edition?: string;
+    year?: number;
+    language?: string;
+    isbn?: string;
+    publisher?: string;
+    tags?: string[];
+    web?: BookWebOrigin;
+    [key: string]: unknown;
+}
+export interface LibraryBook {
+    id: number;
+    uid: string;
+    /** Undefined for a book in the trash. */
+    shelfId?: number;
+    collectionId: number;
+    title: string;
+    description: string;
+    metadata: BookMetadata;
+    createdAt: string;
+    updatedAt: string;
+    trashedAt?: string;
+    /** Where a trashed book stood. */
+    trashedFrom?: {
+        section: string;
+        shelf: string;
+    };
+    sources: number;
+    passages: number;
+}
+export type BookSourceKind = "file" | "web" | "text";
+export interface BookSource {
+    id: number;
+    bookId: number;
+    kind: BookSourceKind;
+    sha256: string;
+    name: string;
+    /** The source itself, relative to the book's folder. */
+    stored?: string;
+    /** Its text, relative to the book's folder. */
+    textFile?: string;
+    url?: string;
+    bytes: number;
+    addedAt: string;
+    documentId?: number;
+    fingerprint?: number[];
+    metadata: Record<string, unknown>;
+    removedAt?: string;
+}
+export interface BookImage {
+    id: number;
+    bookId: number;
+    sourceId: number;
+    sha256: string;
+    /** Relative to the book's folder. */
+    file: string;
+    mediaType: string;
+    bytes: number;
+    width?: number;
+    height?: number;
+    /** Where in the source it was: `page 7`, `chapter 2`. */
+    origin?: string;
+    /** What a vision model said it shows, or failing that its caption in the book. */
+    description?: string;
+    /** A vision model has described it: `description` is not just its caption. */
+    described: boolean;
+}
+export interface SourceImageInput {
+    /** The name the source's text links it by: `images/<file>`. */
+    file: string;
+    data: Uint8Array;
+    mediaType: string;
+    width?: number;
+    height?: number;
+    origin?: string;
+    description?: string;
+    /** `description` is a vision model's, not the caption the book gave it. */
+    described?: boolean;
+}
+export interface AddSourceInput {
+    kind: BookSourceKind;
+    /** A file name, or a page's title. */
+    name: string;
+    /** For a file: where it is now. It is copied into the book. */
+    path?: string;
+    url?: string;
+    /** The source as text or markdown: what is searched. */
+    text: string;
+    title?: string;
+    metadata?: Record<string, unknown>;
+    images?: readonly SourceImageInput[];
+    /** A source of this book it takes the place of. */
+    replaces?: number;
+}
+export interface AddSourceResult {
+    source: BookSource;
+    outcome: "added" | "replaced" | "unchanged";
+    passages: number;
+    images: number;
+}
+export interface BookMatch {
+    book: LibraryBook;
+    source?: BookSource;
+    reason: "same file" | "same link" | "same isbn" | "same title and author" | "same title" | "similar text";
+    /** For "similar text": the share of the text the two have in common, 0 to 1. */
+    similarity?: number;
+}
+export interface FindSimilarInput {
+    sha256?: string;
+    url?: string;
+    title?: string;
+    authors?: readonly string[];
+    isbn?: string;
+    fingerprint?: readonly number[];
+    /** Text similarity below this is not reported. @default 0.3 */
+    minSimilarity?: number;
+}
+export type ExportScope = {
+    library: true;
+} | {
+    sectionId: number;
+} | {
+    shelfId: number;
+} | {
+    bookId: number;
+};
+export interface ExportResult {
+    file: string;
+    books: number;
+    files: number;
+    bytes: number;
+}
+export interface ImportOptions {
+    settings?: LibrarySettings;
+    /** Put every book on this shelf, whatever shelf the archive says. */
+    shelfId?: number;
+    /**
+     * A book already here (the same book, by its id): leave it, replace it,
+     * or bring the archive's in beside it as a copy. @default "skip"
+     */
+    existing?: "skip" | "replace" | "copy";
+}
+export interface ImportResult {
+    imported: {
+        title: string;
+        section: string;
+        shelf: string;
+    }[];
+    skipped: {
+        title: string;
+        reason: string;
+    }[];
+}
+export interface CatalogueProblem {
+    book?: string;
+    problem: string;
+}
+/** A title as it is compared: lower case, letters and digits only. */
+export declare function normalizeTitle(title: string): string;
+export declare function normalizeIsbn(isbn: string): string;
+export declare class Catalogue {
+    private readonly library;
+    private readonly db;
+    private readonly booksDirectory;
+    constructor(library: Library);
+    private one;
+    private all;
+    bookDirectory(book: Pick<LibraryBook, "uid">): string;
+    /**
+     * A Library from before it had shelves: each of its collections becomes a
+     * shelf in "General" holding one book of that name. The documents stay
+     * indexed as they are; their files were never copied in, so the book
+     * records where they were.
+     */
+    private adoptLegacyCollections;
+    sections(): LibrarySection[];
+    findSection(name: string): LibrarySection | undefined;
+    /** A shelf by name; in one section when that is given, and otherwise wherever it is the only one of that name. */
+    findShelf(name: string, section?: string): LibraryShelf | undefined;
+    shelf(shelfId: number): (LibraryShelf & {
+        section: string;
+    }) | undefined;
+    ensureSection(name: string, description?: string): LibrarySection;
+    updateSection(sectionId: number, change: {
+        name?: string;
+        description?: string;
+    }): void;
+    /** Remove a section and its shelves. Their books go to the trash. */
+    deleteSection(sectionId: number): number;
+    ensureShelf(sectionId: number, name: string, description?: string): LibraryShelf;
+    updateShelf(shelfId: number, change: {
+        name?: string;
+        description?: string;
+        sectionId?: number;
+    }): void;
+    /** Remove a shelf. Its books go to the trash; resolves with how many. */
+    deleteShelf(shelfId: number): number;
+    private toBook;
+    private static readonly BOOK_COLUMNS;
+    /** The books on a shelf, or all of them; the trash only when asked for. */
+    books(filter?: {
+        shelfId?: number;
+        trashed?: boolean;
+    }): LibraryBook[];
+    book(bookId: number): LibraryBook | undefined;
+    bookByUid(uid: string): LibraryBook | undefined;
+    /**
+     * A book by what the model can name it with: its number (`#12`), its id,
+     * or its title. Several books of one title come back together, for the
+     * caller to say which.
+     */
+    findBooks(reference: string, options?: {
+        trashed?: boolean;
+    }): LibraryBook[];
+    /** The collections search may read: every book not in the trash, or those of the shelves named. */
+    searchableCollections(shelfIds?: readonly number[]): number[];
+    /** Where a collection's passages are from: "Section / Shelf / Title". */
+    collectionLabels(): Map<number, string>;
+    createBook(input: {
+        shelfId: number;
+        title: string;
+        description?: string;
+        metadata?: BookMetadata;
+        uid?: string;
+        createdAt?: string;
+    }): LibraryBook;
+    /** Change what a book says of itself. `metadata` is merged; a key set to null is removed. */
+    updateBook(bookId: number, change: {
+        title?: string;
+        description?: string;
+        metadata?: Record<string, unknown>;
+        shelfId?: number;
+    }): LibraryBook;
+    /** To the trash: out of every search, and gone for good after thirty days. */
+    trashBook(bookId: number): void;
+    /** Out of the trash, onto the shelf it was on (made again if it is gone) or the one given. */
+    restoreBook(bookId: number, shelfId?: number): LibraryBook;
+    /** Delete a book for good: its passages, its vectors and its folder. */
+    purgeBook(bookId: number): Promise<void>;
+    /** Empty the trash of what has been in it past its days. `days: 0` empties it. */
+    purgeTrash(days?: number, at?: Date): Promise<{
+        books: number;
+        sources: number;
+    }>;
+    private toSource;
+    sources(bookId: number, options?: {
+        removed?: boolean;
+    }): BookSource[];
+    source(sourceId: number): BookSource | undefined;
+    images(bookId: number): BookImage[];
+    /** The name a source is indexed under: its own, made unique within the book. */
+    private documentName;
+    /**
+     * Add a source to a book: the file is copied in under its sha256, its
+     * text and pictures are written beside it, and the text is indexed. The
+     * same content already in the book is left alone; a page of the same
+     * link with other content takes the old one's place.
+     */
+    addSource(bookId: number, input: AddSourceInput, settings?: LibrarySettings): Promise<AddSourceResult>;
+    /**
+     * Take a source out of its book. It is no longer searched, and its files
+     * stay in the book's folder for thirty days, where `restoreSource` finds
+     * them.
+     */
+    removeSource(sourceId: number): Promise<void>;
+    restoreSource(sourceId: number, settings?: LibrarySettings): Promise<BookSource>;
+    /** Index a source's text from its file in the book's folder. */
+    private indexSource;
+    /** Remove a source for good: its passages, its vectors, its rows and its files. */
+    private dropSource;
+    /** Give a picture its description, or another one. */
+    describeImage(imageId: number, description: string): void;
+    /**
+     * Give pictures of one source their descriptions after the fact: each is
+     * kept with its picture, written into the source's text where the picture
+     * is linked, and the source is indexed again so the words can be found.
+     *
+     * This is what adding a book does when a vision model is set; it lets a
+     * book added without one be completed without reading the book again.
+     */
+    describeSourceImages(sourceId: number, described: readonly {
+        imageId: number;
+        description: string;
+        alt: string;
+    }[], settings?: LibrarySettings): Promise<{
+        described: number;
+        passages: number;
+    }>;
+    /**
+     * Move every source of one book into another, and trash the emptied one.
+     * A source the other already holds is left where it is.
+     */
+    mergeBooks(fromId: number, intoId: number, settings?: LibrarySettings): Promise<{
+        moved: number;
+        alreadyThere: number;
+    }>;
+    /**
+     * The books that may be this one: the same file, the same link, the same
+     * ISBN, the same title, or text that is largely the same. Books in the
+     * trash are included, and say so.
+     */
+    findSimilar(input: FindSimilarInput): BookMatch[];
+    private bookFile;
+    /** Write `book.json`: what the folder is, for a person and for import. */
+    writeBookFile(bookId: number): void;
+    private rewriteBookFiles;
+    /**
+     * Write the Library, a section, a shelf or one book to a file: the books'
+     * folders as they are, and where each stood. Vectors are left out; they
+     * are made again where the file is read in, with the model set there.
+     */
+    export(scope: ExportScope, file: string): Promise<ExportResult>;
+    /**
+     * Read an exported file in. Sections and shelves are made where they are
+     * missing; a book already here is left alone unless told otherwise.
+     */
+    import(file: string, options?: ImportOptions): Promise<ImportResult>;
+    private importBook;
+    /** What is wrong with the Library as it stands: missing files, books with nothing in them. */
+    problems(): CatalogueProblem[];
+}

package/dist/services/retrieval/chunker.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Splits a document's text into the chunks that are indexed.
+ *
+ * Every chunk is a contiguous slice of the source, so it carries the offsets
+ * it came from and can be shown in place. The steps, in order:
+ *
+ * 1. With `markdownHeaders`, the text is first cut into sections at its
+ *    markdown headers, and each chunk remembers the headers above it.
+ * 2. A section larger than `size` is cut at the most natural boundary that
+ *    is small enough: paragraphs, then lines, then sentences, then words,
+ *    and only then mid-word.
+ * 3. Neighbouring pieces are packed into chunks of at most `size`, each
+ *    starting up to `overlap` before the end of the one before.
+ * 4. With `minSize`, a chunk smaller than that is merged into its neighbour
+ *    when the result still fits, forward first and then backward.
+ *
+ * Sizes are counted in characters, or in estimated tokens (four characters
+ * each) unless a real counter is passed as `measure`.
+ */
+export interface ChunkOptions {
+    /** Largest chunk. Default 1500. */
+    size?: number;
+    /** Shared between neighbouring chunks of one section. Default 100. */
+    overlap?: number;
+    /** Chunks smaller than this are merged when possible. 0 turns it off. Default 0. */
+    minSize?: number;
+    /** What `size`, `overlap` and `minSize` count. Default "characters". */
+    unit?: "characters" | "tokens";
+    /** Cut at markdown headers first, and record them. Default true. */
+    markdownHeaders?: boolean;
+    /** A real counter, replacing the estimate for `unit`. */
+    measure?: (text: string) => number;
+}
+export interface TextChunk {
+    /** Position among the document's chunks, from 0. */
+    index: number;
+    text: string;
+    /** Offsets of `text` in the source: `source.slice(start, end) === text`. */
+    start: number;
+    end: number;
+    /** The markdown headers above the chunk's start, outermost first. */
+    headings: string[];
+}
+export declare function chunkText(source: string, options?: ChunkOptions): TextChunk[];

package/dist/services/retrieval/embedding-client.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Embedding and reranking over HTTP, for the Library and the memory store.
+ *
+ * One wire format each, the ones every server in use speaks:
+ *
+ * - embeddings: OpenAI's `POST <base>/embeddings` with `{model, input: []}`,
+ *   answered by `{data: [{index, embedding}]}`. Ollama, llama.cpp, opencoti,
+ *   LM Studio, vLLM, TEI and OpenAI itself all serve it.
+ * - reranking: `POST <base>/rerank` with `{model, query, documents, top_n}`,
+ *   answered by `{results: [{index, relevance_score}]}` (Jina's and Cohere's
+ *   shape; llama.cpp with `--reranking`, vLLM and Infinity serve it). TEI's
+ *   bare array of `{index, score}` is read too.
+ *
+ * A server that is busy is not a failure: 429 and 5xx are retried with a
+ * growing wait before the error is passed on.
+ */
+export interface RetrievalEndpoint {
+    /** With or without a trailing `/v1`; `/v1` is added when the path has none. */
+    baseUrl: string;
+    model: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+}
+export interface EmbedOptions {
+    /** Texts per request. Default 64. */
+    batchSize?: number;
+    /** Requests in flight at once. 0 means no limit of ours. Default 4. */
+    concurrency?: number;
+    /**
+     * Text put in front of every input. Instruction-tuned embedders want one
+     * for queries and usually none for documents.
+     */
+    prefix?: string;
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+    /** Attempts per request before a busy or failing server is reported. Default 4. */
+    maxAttempts?: number;
+    /** First wait between attempts, doubled each time. Default 500. */
+    retryDelayMs?: number;
+}
+export interface EmbedResult {
+    vectors: Float32Array[];
+    dimension: number;
+    /** The model as the server named it, when it did. */
+    model?: string;
+    promptTokens?: number;
+}
+export interface RerankOptions {
+    /** Documents per request. Default 32. */
+    batchSize?: number;
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+    maxAttempts?: number;
+    retryDelayMs?: number;
+}
+export declare class RetrievalEndpointError extends Error {
+    readonly status?: number | undefined;
+    readonly body?: string | undefined;
+    constructor(message: string, status?: number | undefined, body?: string | undefined);
+}
+/** `http://host:1234` and `http://host:1234/v1/` both become `http://host:1234/v1`. */
+export declare function resolveRetrievalBaseUrl(baseUrl: string): string;
+/**
+ * Whether the server refused an input for its size. llama.cpp reports that
+ * as a 500, which would otherwise read as a server worth asking again.
+ */
+export declare function isInputTooLarge(error: unknown): boolean;
+/**
+ * Embed texts, in the order given. Every vector has the same length, and
+ * there is exactly one per text, or this throws: a short or ragged answer
+ * written to an index is worse than no answer.
+ */
+export declare function embedTexts(endpoint: RetrievalEndpoint, texts: readonly string[], options?: EmbedOptions): Promise<EmbedResult>;
+/**
+ * Score each document against the query. One score per document, in the
+ * order the documents were given; higher is more relevant.
+ */
+export declare function rerankDocuments(endpoint: RetrievalEndpoint, query: string, documents: readonly string[], options?: RerankOptions): Promise<number[]>;

package/dist/services/retrieval/fingerprint.d.ts

@@ -0,0 +1,21 @@
+/**
+ * A text's fingerprint: a few numbers that say how much of two texts is the
+ * same, without keeping either.
+ *
+ * This is how the Library tells that a file is a book it already has under
+ * another name or in another format, and how it tells a new edition (much
+ * the same, not all) from a copy (all but the same). It needs no model.
+ *
+ * MinHash over five-word shingles: the share of positions where two
+ * fingerprints agree estimates the share of five-word runs the texts have in
+ * common.
+ */
+export declare const FINGERPRINT_SIZE = 64;
+/**
+ * The fingerprint of a text. Markup, case, punctuation and line breaks are
+ * not part of it, so the same book read out of a PDF and out of an EPUB come
+ * out close. Empty for a text of fewer than five words.
+ */
+export declare function textFingerprint(text: string): number[];
+/** How much of two texts is the same, 0 to 1, from their fingerprints. */
+export declare function fingerprintSimilarity(a: readonly number[], b: readonly number[]): number;