@cerebriline/core 4.100.244 → 4.100.246

package/dist/runtime/host/local-runtime-host.d.ts

@@ -59,6 +59,8 @@ export declare class LocalRuntimeHost implements RuntimeHost {
     private readonly sessions;
     private readonly manifestMutationQueues;
     private readonly usageBySession;
+    /** Per session: the listener for its background requests' waits. */
+    private readonly sideCallUnsubscribes;
     private readonly aggregateUsageBySession;
     private readonly subAgentStarts;
     private readonly pendingPromptsController;

package/dist/services/retrieval/code-index.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Search a workspace's code by meaning.
+ *
+ * `search_codebase` is a regex: it finds a name the model already knows.
+ * `ask_lsp` resolves a symbol the model already has. Neither answers "where
+ * does this app check a user's password" when no word of the question is in
+ * the code. An embedding model does, and the Library already has everything
+ * that needs: chunking, a keyword index, vectors, the merge of the two and a
+ * reranker.
+ *
+ * So the code index is a second Library, kept apart from the user's books
+ * under `<data dir>/code-index`, with one collection per workspace and one
+ * document per source file. Nothing here knows about embeddings beyond what
+ * it hands on.
+ *
+ * It is opt-in per workspace: indexing sends every source file to the
+ * embedder, which on a large repository is real work for a GPU.
+ */
+import { type LibrarySettings } from "@cerebriline/shared";
+import type { RetrievalEndpoint } from "./embedding-client";
+import { type EmbedPendingResult, Library } from "./library";
+import type { RetrieveResult } from "./retrieve";
+export declare function resolveCodeIndexDirectory(): string;
+/** A source file larger than this is a data file or a bundle, not code to search. */
+export declare const CODE_INDEX_MAX_FILE_BYTES: number;
+/** Files indexed per workspace, at most. Past it the rest is left out and said so. */
+export declare const CODE_INDEX_MAX_FILES = 20000;
+export declare function isIndexableCodePath(relativePath: string): boolean;
+export interface CodeIndexOptions {
+    /** @default <data dir>/code-index */
+    directory?: string;
+    /** Where LanceDB is installed. @default the Library's */
+    runtimeDirectory?: string;
+    fetch?: typeof fetch;
+}
+export interface CodeSyncOptions {
+    settings?: LibrarySettings;
+    /** The workspace's files, relative with forward slashes. @default ripgrep's list, which honours .gitignore */
+    listFiles?: (root: string) => Promise<Iterable<string>>;
+    onProgress?: (done: number, total: number) => void;
+    signal?: AbortSignal;
+}
+export interface CodeSyncResult {
+    /** Files in the index after the run. */
+    files: number;
+    added: number;
+    updated: number;
+    removed: number;
+    /** Files left out: too large, binary, generated, or unreadable. */
+    skipped: number;
+    /** Set when the workspace has more files than are indexed. */
+    truncated?: number;
+}
+export interface CodeEmbedOptions {
+    embedding: RetrievalEndpoint;
+    settings?: LibrarySettings;
+    /** Download LanceDB when it is not installed. */
+    install?: boolean;
+    onProgress?: (documentsDone: number, documents: number) => void;
+    signal?: AbortSignal;
+}
+export interface CodeSearchOptions {
+    settings?: LibrarySettings;
+    embedding?: RetrievalEndpoint;
+    reranker?: RetrievalEndpoint;
+    /** Passages returned. @default 8 */
+    limit?: number;
+    /** How the folder's files are read for line numbers. @default from disk */
+    readFile?: (absolutePath: string) => Promise<string | undefined>;
+    signal?: AbortSignal;
+}
+export interface CodeHit {
+    /** Relative to the workspace, forward slashes. */
+    path: string;
+    /** 1-based, inclusive. Absent when the file has changed since it was indexed. */
+    startLine?: number;
+    endLine?: number;
+    text: string;
+    rerankScore?: number;
+    similarity?: number;
+}
+export interface CodeSearchResult {
+    hits: CodeHit[];
+    mode: RetrieveResult["mode"];
+    reranked: boolean;
+    notes: string[];
+}
+export interface CodeIndexStatus {
+    indexed: boolean;
+    files: number;
+    chunks: number;
+}
+/** One name for a workspace however its path was spelled. */
+export declare function codeCollectionName(root: string): string;
+export declare class CodeIndex {
+    readonly library: Library;
+    constructor(options?: CodeIndexOptions);
+    private collection;
+    status(root: string): CodeIndexStatus;
+    /**
+     * Bring the index in line with the files on disk: new and changed files
+     * are read and chunked, files that are gone are dropped. A file whose size
+     * and modification time are what they were is not read again.
+     *
+     * Searchable by keyword when this returns; `embed` adds the vectors.
+     */
+    sync(root: string, options?: CodeSyncOptions): Promise<CodeSyncResult>;
+    /** Give vectors to every indexed file that has none for this embedding model. */
+    embed(root: string, options: CodeEmbedOptions): Promise<EmbedPendingResult>;
+    search(root: string, query: string, options?: CodeSearchOptions): Promise<CodeSearchResult>;
+    /** Forget a workspace: its files, chunks and vectors. */
+    remove(root: string): Promise<boolean>;
+    close(): Promise<void>;
+}
+/** The code index of the data folder, opened once. */
+export declare function sharedCodeIndex(): CodeIndex;
+export declare function closeSharedCodeIndex(): Promise<void>;

package/dist/services/retrieval/firecrawl.d.ts

@@ -21,6 +21,11 @@ export interface ScrapedPage {
     sha256: string;
     /** The links on the page, when they were asked for. */
     links?: string[];
+    /**
+     * The whole page as the browser held it after its scripts ran, when it
+     * was asked for: every tag, nothing cleaned out.
+     */
+    rawHtml?: string;
 }
 export interface ScrapeFailure {
     url: string;
@@ -34,6 +39,15 @@ export interface CrawlOptions {
     /** Only paths matching these (regular expressions), when given. */
     includePaths?: readonly string[];
     excludePaths?: readonly string[];
+    /**
+     * Follow links anywhere on the site. Without it a crawl stays below the
+     * address it starts from: from `/docs/intro` it never reaches `/docs/api`.
+     */
+    entireDomain?: boolean;
+    /** Also give each page's links, to tell what a crawl left unread. */
+    links?: boolean;
+    /** Also give each page whole, as rendered (`ScrapedPage.rawHtml`). */
+    rawHtml?: boolean;
     signal?: AbortSignal;
     fetch?: typeof fetch;
     /** How long to wait for the crawl. @default 20 minutes */
@@ -62,6 +76,7 @@ export declare function htmlToMarkdown(html: string): Promise<string>;
 /** Read one page. */
 export declare function scrapePage(endpoint: ScrapeEndpoint, url: string, options?: RequestOptions & {
     links?: boolean;
+    rawHtml?: boolean;
 }): Promise<ScrapedPage>;
 /** Read a page and the pages it links to, as deep and as many as `options` allow. */
 export declare function crawlSite(endpoint: ScrapeEndpoint, url: string, options: CrawlOptions): Promise<CrawlResult>;

package/dist/services/retrieval/web-mirror.d.ts

@@ -0,0 +1,101 @@
+/**
+ * A site's pages and the files they need, saved as they are.
+ *
+ * The scraping endpoint reads pages in a browser and gives back their content;
+ * it does not give back a stylesheet, a script, a picture or a font. Those are
+ * fetched here, straight from the site, by the machine this runs on.
+ *
+ * Nothing is rewritten. A page is saved as the site served it, byte for byte,
+ * and beside it as the browser held it once its scripts had run -- on a site
+ * built by JavaScript the first is a shell of a few lines and the second is
+ * the page. Links inside them still point where they pointed.
+ */
+export interface MirrorPage {
+    url: string;
+    /** The page as rendered, when the scraper gave it. */
+    rawHtml?: string;
+}
+export interface MirrorLimits {
+    /** Files fetched besides the pages, at most. @default 2000 */
+    maxAssets?: number;
+    /** One file's size, at most. @default 25 MB */
+    maxAssetBytes?: number;
+    /** Everything fetched, at most. @default 300 MB */
+    maxTotalBytes?: number;
+    /** How many requests run at once. @default 6 */
+    concurrency?: number;
+    /** How long one request may take. @default 30 s */
+    timeoutMs?: number;
+}
+export interface MirrorOptions extends MirrorLimits {
+    /** The folder everything is written under. */
+    root: string;
+    fetch?: typeof fetch;
+    signal?: AbortSignal;
+    onProgress?: (done: number, known: number) => void;
+    /**
+     * The file already saved for an address, from an earlier crawl into the
+     * same folder: it is not fetched again.
+     */
+    have?: (url: string) => string | undefined;
+    /** Files to fetch besides those the pages name: what an earlier crawl left. */
+    also?: readonly string[];
+}
+export interface MirroredPage {
+    url: string;
+    /** As the site served it; absent when the direct request failed. */
+    served?: string;
+    /** As the browser rendered it; absent when the scraper gave none. */
+    rendered?: string;
+    /** Why there is no `served`. */
+    problem?: string;
+}
+export interface MirrorReport {
+    pages: MirroredPage[];
+    /** Files saved besides the pages, by their path under the folder. */
+    assets: {
+        url: string;
+        file: string;
+        bytes: number;
+    }[];
+    failed: {
+        url: string;
+        reason: string;
+    }[];
+    /** Files left out because a limit was reached. */
+    skipped: number;
+    /** Their addresses, for a later crawl to fetch. */
+    left: string[];
+    /** How many of them the limit of files stopped, and how many the total size. */
+    leftForCount: number;
+    leftForSize: number;
+    /** Files over the size one file may have; they are in `failed` too. */
+    tooLarge: number;
+    /** Files an earlier crawl had already saved. */
+    reused: number;
+    bytes: number;
+}
+/**
+ * Where a page or a file of a site goes under the folder: its host, then its
+ * path as the site has it. A page always ends in `.html`; a path ending in `/`
+ * is that folder's `index.html`. A page's query is part of its name, since two
+ * queries are two pages; a file's is not, since it is nearly always a cache
+ * stamp on one file.
+ */
+export declare function mirrorPath(url: string, kind: "page" | "asset", contentType?: string): string | undefined;
+/** `X.html` as `X.rendered.html`. */
+export declare function renderedPath(served: string): string;
+/** The files a stylesheet needs: what it imports, and every `url(...)`. */
+export declare function cssLinks(css: string, base: string): string[];
+/**
+ * The files a page needs to look and behave as it does: stylesheets, scripts,
+ * pictures, fonts, icons, media. Not the pages it links to.
+ */
+export declare function assetLinks(html: string, pageUrl: string): string[];
+/**
+ * Save `pages` and every file they need under `options.root`.
+ *
+ * Never throws for a page or a file that could not be had: each is in the
+ * report with its reason. Throws only when the run is aborted.
+ */
+export declare function mirrorSite(pages: readonly MirrorPage[], options: MirrorOptions): Promise<MirrorReport>;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@cerebriline/core",
 	"description": "Cerebriline Core SDK for Node Runtime",
-	"version": "4.100.244",
+	"version": "4.100.246",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/mann1x/cline.git",
@@ -69,9 +69,9 @@
 		"ws": "^8.20.0",
 		"yaml": "^2.8.2",
 		"zod": "^4.3.6",
-		"@cerebriline/agents": "4.100.244",
-		"@cerebriline/llms": "4.100.244",
-		"@cerebriline/shared": "4.100.244"
+		"@cerebriline/agents": "4.100.246",
+		"@cerebriline/llms": "4.100.246",
+		"@cerebriline/shared": "4.100.246"
 	},
 	"peerDependencies": {
 		"posthog-node": "^5.8.0"