@cerebriline/core 4.100.245 → 4.100.246

package/dist/runtime/host/local-runtime-host.d.ts

@@ -59,6 +59,8 @@ export declare class LocalRuntimeHost implements RuntimeHost {
     private readonly sessions;
     private readonly manifestMutationQueues;
     private readonly usageBySession;
+    /** Per session: the listener for its background requests' waits. */
+    private readonly sideCallUnsubscribes;
     private readonly aggregateUsageBySession;
     private readonly subAgentStarts;
     private readonly pendingPromptsController;

package/dist/services/retrieval/code-index.d.ts

@@ -65,6 +65,8 @@ export interface CodeSearchOptions {
     reranker?: RetrievalEndpoint;
     /** Passages returned. @default 8 */
     limit?: number;
+    /** How the folder's files are read for line numbers. @default from disk */
+    readFile?: (absolutePath: string) => Promise<string | undefined>;
     signal?: AbortSignal;
 }
 export interface CodeHit {

package/dist/services/retrieval/firecrawl.d.ts

@@ -21,6 +21,11 @@ export interface ScrapedPage {
     sha256: string;
     /** The links on the page, when they were asked for. */
     links?: string[];
+    /**
+     * The whole page as the browser held it after its scripts ran, when it
+     * was asked for: every tag, nothing cleaned out.
+     */
+    rawHtml?: string;
 }
 export interface ScrapeFailure {
     url: string;
@@ -34,6 +39,15 @@ export interface CrawlOptions {
     /** Only paths matching these (regular expressions), when given. */
     includePaths?: readonly string[];
     excludePaths?: readonly string[];
+    /**
+     * Follow links anywhere on the site. Without it a crawl stays below the
+     * address it starts from: from `/docs/intro` it never reaches `/docs/api`.
+     */
+    entireDomain?: boolean;
+    /** Also give each page's links, to tell what a crawl left unread. */
+    links?: boolean;
+    /** Also give each page whole, as rendered (`ScrapedPage.rawHtml`). */
+    rawHtml?: boolean;
     signal?: AbortSignal;
     fetch?: typeof fetch;
     /** How long to wait for the crawl. @default 20 minutes */
@@ -62,6 +76,7 @@ export declare function htmlToMarkdown(html: string): Promise<string>;
 /** Read one page. */
 export declare function scrapePage(endpoint: ScrapeEndpoint, url: string, options?: RequestOptions & {
     links?: boolean;
+    rawHtml?: boolean;
 }): Promise<ScrapedPage>;
 /** Read a page and the pages it links to, as deep and as many as `options` allow. */
 export declare function crawlSite(endpoint: ScrapeEndpoint, url: string, options: CrawlOptions): Promise<CrawlResult>;

package/dist/services/retrieval/web-mirror.d.ts

@@ -0,0 +1,101 @@
+/**
+ * A site's pages and the files they need, saved as they are.
+ *
+ * The scraping endpoint reads pages in a browser and gives back their content;
+ * it does not give back a stylesheet, a script, a picture or a font. Those are
+ * fetched here, straight from the site, by the machine this runs on.
+ *
+ * Nothing is rewritten. A page is saved as the site served it, byte for byte,
+ * and beside it as the browser held it once its scripts had run -- on a site
+ * built by JavaScript the first is a shell of a few lines and the second is
+ * the page. Links inside them still point where they pointed.
+ */
+export interface MirrorPage {
+    url: string;
+    /** The page as rendered, when the scraper gave it. */
+    rawHtml?: string;
+}
+export interface MirrorLimits {
+    /** Files fetched besides the pages, at most. @default 2000 */
+    maxAssets?: number;
+    /** One file's size, at most. @default 25 MB */
+    maxAssetBytes?: number;
+    /** Everything fetched, at most. @default 300 MB */
+    maxTotalBytes?: number;
+    /** How many requests run at once. @default 6 */
+    concurrency?: number;
+    /** How long one request may take. @default 30 s */
+    timeoutMs?: number;
+}
+export interface MirrorOptions extends MirrorLimits {
+    /** The folder everything is written under. */
+    root: string;
+    fetch?: typeof fetch;
+    signal?: AbortSignal;
+    onProgress?: (done: number, known: number) => void;
+    /**
+     * The file already saved for an address, from an earlier crawl into the
+     * same folder: it is not fetched again.
+     */
+    have?: (url: string) => string | undefined;
+    /** Files to fetch besides those the pages name: what an earlier crawl left. */
+    also?: readonly string[];
+}
+export interface MirroredPage {
+    url: string;
+    /** As the site served it; absent when the direct request failed. */
+    served?: string;
+    /** As the browser rendered it; absent when the scraper gave none. */
+    rendered?: string;
+    /** Why there is no `served`. */
+    problem?: string;
+}
+export interface MirrorReport {
+    pages: MirroredPage[];
+    /** Files saved besides the pages, by their path under the folder. */
+    assets: {
+        url: string;
+        file: string;
+        bytes: number;
+    }[];
+    failed: {
+        url: string;
+        reason: string;
+    }[];
+    /** Files left out because a limit was reached. */
+    skipped: number;
+    /** Their addresses, for a later crawl to fetch. */
+    left: string[];
+    /** How many of them the limit of files stopped, and how many the total size. */
+    leftForCount: number;
+    leftForSize: number;
+    /** Files over the size one file may have; they are in `failed` too. */
+    tooLarge: number;
+    /** Files an earlier crawl had already saved. */
+    reused: number;
+    bytes: number;
+}
+/**
+ * Where a page or a file of a site goes under the folder: its host, then its
+ * path as the site has it. A page always ends in `.html`; a path ending in `/`
+ * is that folder's `index.html`. A page's query is part of its name, since two
+ * queries are two pages; a file's is not, since it is nearly always a cache
+ * stamp on one file.
+ */
+export declare function mirrorPath(url: string, kind: "page" | "asset", contentType?: string): string | undefined;
+/** `X.html` as `X.rendered.html`. */
+export declare function renderedPath(served: string): string;
+/** The files a stylesheet needs: what it imports, and every `url(...)`. */
+export declare function cssLinks(css: string, base: string): string[];
+/**
+ * The files a page needs to look and behave as it does: stylesheets, scripts,
+ * pictures, fonts, icons, media. Not the pages it links to.
+ */
+export declare function assetLinks(html: string, pageUrl: string): string[];
+/**
+ * Save `pages` and every file they need under `options.root`.
+ *
+ * Never throws for a page or a file that could not be had: each is in the
+ * report with its reason. Throws only when the run is aborted.
+ */
+export declare function mirrorSite(pages: readonly MirrorPage[], options: MirrorOptions): Promise<MirrorReport>;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@cerebriline/core",
 	"description": "Cerebriline Core SDK for Node Runtime",
-	"version": "4.100.245",
+	"version": "4.100.246",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/mann1x/cline.git",
@@ -69,9 +69,9 @@
 		"ws": "^8.20.0",
 		"yaml": "^2.8.2",
 		"zod": "^4.3.6",
-		"@cerebriline/agents": "4.100.245",
-		"@cerebriline/llms": "4.100.245",
-		"@cerebriline/shared": "4.100.245"
+		"@cerebriline/agents": "4.100.246",
+		"@cerebriline/llms": "4.100.246",
+		"@cerebriline/shared": "4.100.246"
 	},
 	"peerDependencies": {
 		"posthog-node": "^5.8.0"