@cerebriline/core 4.100.240 → 4.100.241

package/dist/services/retrieval/memory.d.ts

@@ -0,0 +1,184 @@
+/**
+ * Memory: short notes the model keeps between tasks.
+ *
+ * A note is a fact worth having next time: a decision and its reason, how
+ * this project is built and tested, a preference of the user, a trap that
+ * cost an hour. It is kept by the retrieval core the Library uses, in a
+ * store of its own, so it is searched the same way: by keyword always, and
+ * by meaning when an embedding model is set.
+ *
+ * There are any number of memories, all in the one store: the main one,
+ * which every workspace starts on, and whichever others the user makes --
+ * one for a workspace, one for a client, one imported from a colleague. A
+ * session keeps new notes in exactly one of them and searches as many as
+ * the user allows it; which, is the user's choice per workspace and not
+ * something the model can widen.
+ */
+import { type MemorySettings } from "@cerebriline/shared";
+import type { RetrievalEndpoint } from "./embedding-client";
+import { type EmbedPendingResult, type LibraryOptions, type LibraryVectorSet } from "./library";
+export interface MemoryItem {
+    /** `m12`: what `forget` takes. */
+    id: string;
+    text: string;
+    /** The memory it is kept in. */
+    memory: string;
+    tags: string[];
+    createdAt: string;
+}
+export interface RecalledMemory extends MemoryItem {
+    /** The reranker's score, 0 to 1, when there was one. */
+    relevance?: number;
+    /** Cosine similarity to the query, when the note was found by meaning. */
+    similarity?: number;
+}
+export interface MemoryEndpoints {
+    embedding?: RetrievalEndpoint;
+    reranker?: RetrievalEndpoint;
+}
+/** One of the memories, as the settings panel lists it. */
+export interface MemoryInfo {
+    name: string;
+    /** The main memory: always there, never deleted. */
+    main: boolean;
+    notes: number;
+    /** The workspace it was made for, when it was made for one. */
+    workspace?: string;
+    createdAt: string;
+}
+/** A memory written to a file, to be read back here or on another machine. */
+export interface MemoryExport {
+    format: typeof MEMORY_EXPORT_FORMAT;
+    version: 1;
+    name: string;
+    workspace?: string;
+    exportedAt: string;
+    notes: Array<{
+        text: string;
+        tags: string[];
+        createdAt: string;
+    }>;
+}
+export declare const MEMORY_EXPORT_FORMAT = "cerebriline-memory";
+/** A memory is a note, not a document: longer than this belongs in the Library. */
+export declare const MEMORY_MAX_CHARS = 4000;
+export declare const MEMORY_NAME_MAX_CHARS = 60;
+export declare function resolveMemoryDirectory(): string;
+/** A name as it is kept: trimmed, one line, of a length a list can show. */
+export declare function normalizeMemoryName(name: string): string;
+export declare class Memory {
+    private readonly library;
+    private readonly directory;
+    private migrated;
+    constructor(options?: LibraryOptions);
+    /** Which workspace each memory was made for. Not in the database: it is about the user's folders, not the notes. */
+    private get registryPath();
+    private readRegistry;
+    private writeRegistry;
+    /**
+     * Bring a store made by the first build to this one's names, once: its
+     * `global` collection is the main memory, and each `project:<path>` is a
+     * memory named after the folder, made for that workspace.
+     */
+    private migrate;
+    private freeName;
+    /** Every memory in the store, the main one first. */
+    listMemories(): MemoryInfo[];
+    /** Make a memory. A name already taken is refused, not reused. */
+    createMemory(input: {
+        name: string;
+        workspace?: string;
+    }): MemoryInfo;
+    /**
+     * Give a memory another name. Its notes, their ids and their vectors stay
+     * as they are. The main memory keeps its name: everything starts on it.
+     */
+    renameMemory(from: string, to: string): MemoryInfo;
+    /** Delete a memory and every note in it. The main memory stays. */
+    deleteMemory(name: string): Promise<number>;
+    /** The collections of these memories; a name that is not a memory is left out. */
+    private collections;
+    private toItem;
+    /** Keep a note. The same text in the same memory is kept once. */
+    remember(input: {
+        text: string;
+        /** @default the main memory */
+        memory?: string;
+        tags?: readonly string[];
+        /** For an imported note: when it was first noted. */
+        createdAt?: string;
+        endpoints?: MemoryEndpoints;
+        signal?: AbortSignal;
+    }): Promise<{
+        item: MemoryItem;
+        outcome: "added" | "unchanged";
+        note?: string;
+    }>;
+    /** The notes that best match, best first, from the memories named. */
+    recall(query: string, options?: {
+        /** @default the main memory */
+        memories?: readonly string[];
+        settings?: MemorySettings;
+        endpoints?: MemoryEndpoints;
+        limit?: number;
+        signal?: AbortSignal;
+    }): Promise<{
+        items: RecalledMemory[];
+        notes: string[];
+    }>;
+    /** Every note of the memories named, newest first. */
+    list(options?: {
+        memories?: readonly string[];
+    }): MemoryItem[];
+    /** Remove a note. False when the memories named hold none with that id. */
+    forget(id: string, options?: {
+        memories?: readonly string[];
+    }): Promise<boolean>;
+    /** A memory as a file's content. The notes only: vectors are made again where it is read. */
+    exportMemory(name: string): MemoryExport;
+    /**
+     * Read an exported memory in. Into the memory named, or into one of the
+     * file's own name, made if it is not there. A note already there is left
+     * alone, so reading the same file twice adds nothing.
+     */
+    importMemory(data: unknown, options?: {
+        into?: string;
+        endpoints?: MemoryEndpoints;
+    }): Promise<{
+        memory: string;
+        created: boolean;
+        added: number;
+        unchanged: number;
+        skipped: number;
+    }>;
+    /** How many notes there are, and how many have vectors for this embedding model. */
+    counts(model?: string): {
+        notes: number;
+        embeddedNotes: number;
+    };
+    /** Give vectors to every note that has none for this embedding model. */
+    embedPending(options: {
+        embedding: RetrievalEndpoint;
+        onProgress?: (progress: {
+            done: number;
+            total: number;
+        }) => void;
+        probe?: boolean;
+        signal?: AbortSignal;
+    }): Promise<EmbedPendingResult>;
+    /** The sets of vectors the notes have, one per embedding model and size. */
+    vectorSets(current?: string): Promise<LibraryVectorSet[]>;
+    deleteVectorSet(table: string): Promise<boolean>;
+    /** Record the vector size a model returned, as a search would. */
+    noteDimension(model: string, dimension: number): void;
+    /** Whether notes can be found by meaning here now, and if not, why. */
+    vectorState(): {
+        installed: boolean;
+        unsupported?: string;
+    };
+    close(): Promise<void>;
+}
+/** Closes every Memory this process opened. For shutdown, like `closeSharedLibraries`. */
+export declare function closeSharedMemories(): Promise<void>;
+/** The one Memory of a folder in this process. */
+export declare function sharedMemory(options?: LibraryOptions): Memory;

package/dist/services/retrieval/retrieve.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Retrieval over the Library: keywords, vectors, or both, then a reranker.
+ *
+ * The shape follows open-webui's, which works well in practice: take
+ * candidates from the keyword index and from the vector index, merge the two
+ * rankings by weighted reciprocal rank (the "BM25 weight" slides between
+ * semantic and lexical), keep the top K, and let a reranking model score
+ * those against the query and keep the best few above a threshold.
+ *
+ * Every stage past the keywords is optional. With no embedding model the
+ * search is keywords alone; when an embedder or reranker cannot be reached,
+ * the stages that did run still answer and the result says what was skipped.
+ */
+import { type RetrievalEndpoint } from "./embedding-client";
+import type { LibraryHit, LibraryStore } from "./library-store";
+import type { VectorSearcher } from "./vector-index";
+export interface RetrieveOptions {
+    store: LibraryStore;
+    collectionIds?: readonly number[];
+    /** Where vectors are searched. Left out, the search is keywords alone. */
+    vectors?: VectorSearcher;
+    embedding?: RetrievalEndpoint & {
+        /** Put in front of the query, for embedders trained with an instruction. */
+        queryPrefix?: string;
+    };
+    reranker?: RetrievalEndpoint & {
+        batchSize?: number;
+    };
+    /**
+     * Use the keyword index together with the vectors. Off, a search with
+     * vectors available is semantic alone. Default on.
+     */
+    hybrid?: boolean;
+    /** 0 is semantic alone, 1 lexical alone. Default 0.5. */
+    bm25Weight?: number;
+    /** Also match file names, titles and section headers by keyword. Default on. */
+    enrich?: boolean;
+    /** Chunks kept after merging. Default 5. */
+    topK?: number;
+    /** Chunks kept after reranking. Default 3. */
+    topKReranker?: number;
+    /**
+     * Chunks scoring below this are dropped: the reranker's score (0 to 1)
+     * when there is one, the cosine similarity of a semantic-only search
+     * otherwise. A merged ranking has no score to hold against it. Default 0.
+     */
+    relevanceThreshold?: number;
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+}
+export interface RetrievedHit extends LibraryHit {
+    /** Position in the keyword ranking, from 1, when it was found there. */
+    keywordRank?: number;
+    /** Position in the vector ranking, from 1, when it was found there. */
+    vectorRank?: number;
+    similarity?: number;
+    /** The reranker's score, 0 to 1. */
+    rerankScore?: number;
+}
+export interface RetrieveResult {
+    hits: RetrievedHit[];
+    /** Which rankings the hits came from. */
+    mode: "keyword" | "vector" | "hybrid";
+    reranked: boolean;
+    /** Stages that were asked for and did not run, and why. */
+    notes: string[];
+}
+/**
+ * Rerankers answer on two scales: a probability, or the raw logit behind it
+ * (llama.cpp). Anything outside 0..1 marks the second, and is put through
+ * the logistic function so that one threshold means the same on both.
+ */
+export declare function normalizeRerankScores(scores: readonly number[]): number[];
+/**
+ * Merge rankings by weighted reciprocal rank. Each ranking is a list of ids,
+ * best first; the result is the ids with their merged score, best first.
+ */
+export declare function fuseRankings(rankings: readonly {
+    ids: readonly number[];
+    weight: number;
+}[]): {
+    id: number;
+    score: number;
+}[];
+export declare function retrieve(query: string, options: RetrieveOptions): Promise<RetrieveResult>;

package/dist/services/retrieval/tar-extract.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Unpacks an npm tarball (gzip over tar) as it streams in, without holding
+ * it in memory: the native library the Library downloads is 200 MB and more.
+ *
+ * Only what npm tarballs contain is handled: regular files and directories,
+ * with PAX and GNU long names. Links and devices are skipped. The first path
+ * component (`package/`) is dropped, and a path that would leave the target
+ * folder is refused.
+ */
+/**
+ * Unpack the gzipped tar read from `source` into `root`. Resolves with the
+ * number of files written.
+ */
+export declare function extractTarGz(source: AsyncIterable<Uint8Array>, root: string): Promise<number>;

package/dist/services/retrieval/tar-write.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Writes a gzipped tar file by file, without holding it in memory: an export
+ * of the Library carries the books' own files, and a shelf of them is large.
+ *
+ * What `extractTarGz` reads back: regular files, with a PAX record for a name
+ * past the header's 100 bytes.
+ */
+export interface TarEntry {
+    /** The path inside the archive, with forward slashes. */
+    name: string;
+    /** A file to copy in, */
+    path?: string;
+    /** or the content itself. */
+    data?: string | Uint8Array;
+}
+/**
+ * Write `entries` to `target` as a gzipped tar. Resolves with the number of
+ * files written.
+ */
+export declare function writeTarGz(target: string, entries: Iterable<TarEntry> | AsyncIterable<TarEntry>): Promise<number>;

package/dist/services/retrieval/vector-index.d.ts

@@ -0,0 +1,82 @@
+/**
+ * The Library's vectors, in LanceDB.
+ *
+ * SQLite holds the text and answers keyword queries; this holds one vector
+ * per chunk and answers "what is close to this". The two are joined on the
+ * chunk id. Vectors from different embedding models are not comparable, so
+ * each model and dimension has a table of its own: changing the model leaves
+ * the old vectors where they are and starts a new table.
+ */
+import type { LanceDbRuntime } from "./lancedb-runtime";
+export interface VectorRow {
+    chunkId: number;
+    collectionId: number;
+    documentId: number;
+    vector: Float32Array;
+}
+export interface VectorHit {
+    chunkId: number;
+    /** Cosine similarity, 1 for the same direction. */
+    similarity: number;
+}
+export interface VectorSearchOptions {
+    collectionIds?: readonly number[];
+    /** Default 10. */
+    limit?: number;
+}
+/** What hybrid retrieval needs from a vector index. */
+export interface VectorSearcher {
+    search(model: string, vector: Float32Array, options?: VectorSearchOptions): Promise<VectorHit[]>;
+}
+/**
+ * Below this many rows a table is scanned: exact, and fast enough. Above it
+ * an approximate index is built, which is what keeps a library of gigabytes
+ * answering in milliseconds.
+ */
+export declare const VECTOR_INDEX_MIN_ROWS = 50000;
+export declare function vectorTableName(model: string, dimension: number): string;
+export declare class VectorIndex implements VectorSearcher {
+    private readonly runtime;
+    private readonly db;
+    private readonly tables;
+    private constructor();
+    static open(runtime: LanceDbRuntime, directory: string): Promise<VectorIndex>;
+    /** The tables there are, as `{model slug, dimension}` names. */
+    tableNames(): Promise<string[]>;
+    private table;
+    private createTable;
+    /**
+     * Add vectors. The caller removes a document's old vectors first
+     * (`deleteDocuments`): chunk ids are new each time a document is replaced.
+     */
+    add(model: string, rows: readonly VectorRow[]): Promise<void>;
+    search(model: string, vector: Float32Array, options?: VectorSearchOptions): Promise<VectorHit[]>;
+    count(model: string, dimension?: number): Promise<number>;
+    /**
+     * Remove these documents' vectors from one model's table only. For
+     * embedding a document again: its vectors for other models are not this
+     * run's to touch, and deleting them empties a set the store still counts.
+     */
+    deleteDocumentsIn(model: string, dimension: number, documentIds: readonly number[]): Promise<void>;
+    /** Every set of vectors there is: its table, the size of its vectors, and how many. */
+    sets(): Promise<{
+        table: string;
+        dimension: number;
+        rows: number;
+    }[]>;
+    /** Delete a whole set of vectors. */
+    dropSet(table: string): Promise<boolean>;
+    /** Remove the vectors of these documents, in every table. */
+    deleteDocuments(documentIds: readonly number[]): Promise<void>;
+    deleteCollection(collectionId: number): Promise<void>;
+    private deleteWhere;
+    /**
+     * Housekeeping after a large ingest: compact what the adds wrote, and build
+     * the approximate index once the table is large enough to need one.
+     * Returns whether the table has an index on its vectors afterwards.
+     */
+    optimize(model: string, dimension: number, options?: {
+        minRows?: number;
+    }): Promise<boolean>;
+    close(): void;
+}

package/dist/services/sdd/sdd-engine.d.ts

@@ -0,0 +1,248 @@
+/**
+ * The engine behind the spec-driven skills.
+ *
+ * The plan of a piece of software — what is wanted, the milestones, their
+ * slices, each slice's tasks, and what was proved along the way — is kept in
+ * one SQLite file, `.sdd/sdd.db` in the project. The database is the truth.
+ * The markdown beside it is written from it, for people to read and to
+ * review in a diff; nothing is read back from those files.
+ *
+ * Why a database and not the files: a model that keeps its place by reading
+ * and ticking markdown has to get every read and every edit right, and a
+ * small model does not. Here the order of the work is the engine's. It
+ * refuses a step that is out of turn and says why, and `next()` answers
+ * "what do I do now" with the one step that is due and everything that step
+ * needs.
+ *
+ * The hierarchy and the order of the phases are those of Get Shit Done
+ * (GSD 2, https://getshitdone.help/), which also keeps its state in a
+ * database and dispatches each step itself.
+ */
+export declare const SDD_DIRECTORY = ".sdd";
+export declare const SDD_DATABASE = "sdd.db";
+export type RequirementStatus = "active" | "validated" | "deferred" | "out_of_scope";
+export type MilestoneStatus = "open" | "complete";
+export type SliceStatus = "pending" | "done";
+export type TaskStatus = "pending" | "in_progress" | "done" | "blocked";
+export type Verdict = "pass" | "needs_remediation";
+export type UatResult = "pass" | "fail" | "needs_human";
+export interface SddRequirement {
+    id: string;
+    title: string;
+    description: string;
+    status: RequirementStatus;
+    /** The slice that owns it, as `M001/S01`, once a roadmap names one. */
+    owner?: string;
+}
+export interface SddMilestone {
+    id: string;
+    title: string;
+    context: string;
+    status: MilestoneStatus;
+    requirementsApproved: boolean;
+    roadmapApproved: boolean;
+    verdict?: Verdict;
+    validation?: string;
+    summary?: string;
+}
+export interface SddSlice {
+    milestone: string;
+    id: string;
+    title: string;
+    goal: string;
+    demo: string;
+    depends: string[];
+    status: SliceStatus;
+    research?: string;
+    summary?: string;
+    replans: number;
+    uat: {
+        check: string;
+        result: UatResult;
+        note?: string;
+    }[];
+}
+export interface SddTask {
+    milestone: string;
+    slice: string;
+    id: string;
+    title: string;
+    steps: string;
+    files: string[];
+    verify: string;
+    expect?: string;
+    status: TaskStatus;
+    summary?: string;
+    evidence?: string;
+    blocker?: string;
+}
+export interface SliceInput {
+    title: string;
+    goal?: string;
+    demo?: string;
+    depends?: string[];
+    requirements?: string[];
+}
+export interface TaskInput {
+    title: string;
+    steps: string;
+    files?: string[];
+    verify: string;
+    expect?: string;
+}
+export type SddStep = "discuss_project" | "discuss_milestone" | "approve_requirements" | "plan_milestone" | "approve_roadmap" | "plan_slice" | "execute_task" | "resume_task" | "replan_slice" | "close_slice" | "validate_milestone" | "complete_milestone" | "stuck";
+export interface SddNext {
+    step: SddStep;
+    /** `M001`, `M001/S01` or `M001/S01/T01`: what the step is about. */
+    ref?: string;
+    /** Whether the step is the user's to answer, so auto mode stops here. */
+    needsUser: boolean;
+    /** What to do, with everything the step needs. */
+    instruction: string;
+}
+/** A step asked for out of turn, or with something missing. The message says what to do instead. */
+export declare class SddRuleError extends Error {
+    constructor(message: string);
+}
+export declare const milestoneId: (n: number) => string;
+export declare const sliceId: (n: number) => string;
+export declare const taskId: (n: number) => string;
+export declare class SddEngine {
+    private readonly db;
+    /** The `.sdd` folder. */
+    readonly directory: string;
+    constructor(projectRoot: string);
+    close(): void;
+    private all;
+    private one;
+    private run;
+    private change;
+    project(): {
+        name: string;
+        description: string;
+        agreements: string;
+    } | undefined;
+    requirements(): SddRequirement[];
+    decisions(): {
+        id: string;
+        title: string;
+        choice: string;
+        why: string;
+    }[];
+    knowledge(): string[];
+    captures(): string[];
+    private toMilestone;
+    milestones(): SddMilestone[];
+    /** The milestone being worked on: the first that is not complete. */
+    activeMilestone(): SddMilestone | undefined;
+    private toSlice;
+    slices(milestone: string): SddSlice[];
+    private toTask;
+    tasks(milestone: string, slice: string): SddTask[];
+    journal(limit?: number): {
+        at: string;
+        action: string;
+        ref?: string;
+        text: string;
+    }[];
+    private milestoneOf;
+    /** `S02`, `M001/S02` or nothing (the slice that is due). */
+    private sliceOf;
+    private taskOf;
+    /** The first slice not done whose dependencies are all done. */
+    private dueSlice;
+    setProject(input: {
+        name?: string;
+        description?: string;
+        agreements?: string;
+    }): void;
+    addRequirement(input: {
+        title: string;
+        description?: string;
+        status?: RequirementStatus;
+    }): string;
+    updateRequirement(id: string, patch: {
+        status?: RequirementStatus;
+        title?: string;
+        description?: string;
+    }): void;
+    addDecision(input: {
+        title: string;
+        choice: string;
+        why?: string;
+    }): string;
+    addKnowledge(note: string): void;
+    /** A thought that is not for now, kept so it is not lost. */
+    capture(note: string): void;
+    quick(input: {
+        title: string;
+        summary: string;
+    }): string;
+    addMilestone(input: {
+        title: string;
+        context: string;
+    }): string;
+    /** Change the open milestone's title or context, when the user changes course. */
+    updateMilestone(input: {
+        title?: string;
+        context?: string;
+    }): void;
+    /** Record that the user approved the requirements or the roadmap. */
+    approve(what: "requirements" | "roadmap"): void;
+    private insertSlices;
+    /** The roadmap: the milestone cut into slices. Replaces an unapproved roadmap. */
+    planMilestone(slices: readonly SliceInput[], ref?: string): string[];
+    /** Slices added to an approved roadmap: remediation, or new scope the user agreed to. */
+    addSlices(slices: readonly SliceInput[], ref?: string): string[];
+    /** Take a slice that has not produced anything off the roadmap. */
+    removeSlice(input: {
+        slice: string;
+        reason: string;
+    }): void;
+    private insertTasks;
+    planSlice(input: {
+        slice?: string;
+        research?: string;
+        tasks: readonly TaskInput[];
+    }): string[];
+    /** Replace the tasks that are not done, after a blocker or a plan that proved wrong. */
+    replanSlice(input: {
+        slice?: string;
+        reason: string;
+        tasks: readonly TaskInput[];
+    }): string[];
+    private setTask;
+    startTask(ref?: string): SddTask;
+    completeTask(input: {
+        task?: string;
+        summary: string;
+        evidence: string;
+    }): void;
+    blockTask(input: {
+        task?: string;
+        reason: string;
+    }): void;
+    /** Close a slice: its summary and how its acceptance checks went. */
+    completeSlice(input: {
+        slice?: string;
+        summary: string;
+        uat: readonly {
+            check: string;
+            result: UatResult;
+            note?: string;
+        }[];
+    }): void;
+    validateMilestone(input: {
+        verdict: Verdict;
+        findings: string;
+        remediation?: readonly SliceInput[];
+    }): string[];
+    completeMilestone(summary: string): void;
+    next(): SddNext;
+    status(): string;
+    /**
+     * Write the markdown from the database. Called after every change. The
+     * files are for reading and for review; nothing reads them back.
+     */
+    render(): void;
+}