@cerebriline/core 4.100.244 → 4.100.246

package/dist/extensions/context/capped-thinking.d.ts

@@ -158,6 +158,11 @@ export interface CappedThinkingCondenserConfig {
     /** Absent when no provider is resolved yet; the condenser then stands down. */
     providerConfig?: ProviderConfig;
     summarizer?: CoreCompactionSummarizerConfig;
+    /**
+     * The conversation the notes are written for (`ProviderConfig.sideCallOf`).
+     * Set for the lead only: an agent's note must never close the lead.
+     */
+    sideCallOf?: string;
     logger?: BasicLogger;
 }
 type PrepareTurnResult = {

package/dist/extensions/context/compaction.d.ts

@@ -61,6 +61,12 @@ export interface ContextCompactionPrepareTurnOptions {
      * visibility is. See {@link createCompactionJournal}.
      */
     journal?: CompactionJournal;
+    /**
+     * The conversation the summaries are written for
+     * (`ProviderConfig.sideCallOf`). Set for the lead only: an agent's summary
+     * must never close the lead.
+     */
+    sideCallOf?: string;
 }
 /**
  * The share of the *content* a compaction aims to leave behind.

package/dist/extensions/tools/code-search.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Search of a workspace's code by meaning, as `search_codebase` uses it.
+ *
+ * The index is the user's choice per folder (Settings > Library): building it
+ * sends every source file in the folder to the embedding model. Where the
+ * folder is opted in and an embedding model is set, this is the executor a
+ * host hands to the tools; the index is brought up to date in the background,
+ * and a search made before that is done says so instead of waiting.
+ */
+import { type LibrarySettings } from "@cerebriline/shared";
+import { type CodeIndex, type CodeSearchResult } from "../../services/retrieval/code-index";
+import type { RetrievalEndpoint } from "../../services/retrieval/embedding-client";
+import type { SemanticSearchExecutor } from "./types";
+export interface CodeSearchConfig {
+    settings: LibrarySettings;
+    /** The embedding model, when the user has set one. */
+    embedding?: RetrievalEndpoint;
+    /** The reranking model, when the user has set one. */
+    reranker?: RetrievalEndpoint;
+}
+export interface CodeIndexProgress {
+    /** What the refresh is doing now. */
+    phase: "reading" | "embedding";
+    done: number;
+    total: number;
+}
+export interface CodeIndexRefreshState {
+    running: boolean;
+    progress?: CodeIndexProgress;
+    /** Why the last refresh stopped short, when it did. */
+    problem?: string;
+    /** When the last refresh finished, in epoch milliseconds. */
+    finishedAt?: number;
+}
+export interface CreateCodeSearchOptions {
+    /** Read on every call: the settings can change mid-session. */
+    getConfig: () => CodeSearchConfig;
+    /** @default the shared code index of the data folder */
+    index?: CodeIndex;
+    /**
+     * Download LanceDB, which holds the vectors, when it is not installed. For
+     * a host with no panel to offer the download from. @default false
+     */
+    install?: boolean;
+    log?: (message: string) => void;
+}
+export interface CodeSearch extends SemanticSearchExecutor {
+    /**
+     * Bring a folder's index up to date: read what changed, then embed what has
+     * no vectors. One at a time per folder; a second call joins the first.
+     * Resolves when it is done and never rejects: the state says what happened.
+     */
+    refresh(root: string): Promise<void>;
+    state(root: string): CodeIndexRefreshState;
+}
+/** Whether a folder is searched by meaning: opted in, with a model to embed with. */
+export declare function codeSearchAvailable(config: CodeSearchConfig, root: string): boolean;
+/** The hits as the model reads them: where each passage is, then the passage. */
+export declare function formatCodeHits(result: CodeSearchResult): string;
+export declare function createCodeSearch(options: CreateCodeSearchOptions): CodeSearch;
+/**
+ * The lead's code search as a delegated agent uses it.
+ *
+ * An agent is built without the lead's executors, so that none of them points
+ * its file tools at the real workspace. This one only reads, and what it reads
+ * for line numbers goes through the agent's own copy of the folder: a passage
+ * in a file the agent has edited is given the lines it is on there.
+ */
+export declare function semanticSearchForAgent(lead: SemanticSearchExecutor | undefined, overlay: {
+    read(path: string): Promise<Buffer>;
+} | undefined): SemanticSearchExecutor | undefined;

package/dist/extensions/tools/definitions.d.ts

@@ -6,7 +6,7 @@
 import { type AgentTool, type ShellKind } from "@cerebriline/shared";
 import { type QaCredential } from "./qa-credentials";
 import { type ApplyPatchInput, type AskQuestionInput, type AwkToolInput, type EditFileInput, type ExtractDocumentInput, type FetchWebContentInput, type GrepToolInput, type ReadFileRequest, type ReadFilesInput, type SearchCodebaseInput, type SedToolInput, type SkillsInput, type SubmitInput } from "./schemas";
-import type { ApplyPatchExecutor, AskQuestionExecutor, AwkExecutor, CreateDefaultToolsOptions, DefaultToolsConfig, EditorExecutor, ExtractDocumentExecutor, FileReadExecutor, GrepExecutor, SearchExecutor, SedExecutor, ShellExecutor, SkillsExecutorWithMetadata, ToolOperationResult, VerifySubmitExecutor, WebFetchExecutor } from "./types";
+import type { ApplyPatchExecutor, AskQuestionExecutor, AwkExecutor, CreateDefaultToolsOptions, DefaultToolsConfig, EditorExecutor, ExtractDocumentExecutor, FileReadExecutor, GrepExecutor, SearchExecutor, SedExecutor, SemanticSearchExecutor, ShellExecutor, SkillsExecutorWithMetadata, ToolOperationResult, VerifySubmitExecutor, WebFetchExecutor } from "./types";
 /**
  * What a successful read with no content is told to mean.
  *
@@ -41,7 +41,7 @@ export declare function createReadFilesTool(executor: FileReadExecutor, config?:
  *
  * Performs regex pattern searches across the codebase.
  */
-export declare function createSearchTool(executor: SearchExecutor, config?: Pick<DefaultToolsConfig, "cwd" | "searchTimeoutMs">): AgentTool<SearchCodebaseInput, ToolOperationResult[]>;
+export declare function createSearchTool(executor: SearchExecutor, config?: Pick<DefaultToolsConfig, "cwd" | "searchTimeoutMs">, semantic?: SemanticSearchExecutor["search"]): AgentTool<SearchCodebaseInput, ToolOperationResult[]>;
 /**
  * Build the run_commands tool description for the shell that will actually
  * execute the commands. The shell kind decides the syntax guidance (quoting,

package/dist/extensions/tools/index.d.ts

@@ -16,7 +16,7 @@ export { type BuiltinToolAvailabilityContext, getCoreAcpToolNames, getCoreBuilti
 export { type ApplyPatchInput, ApplyPatchInputSchema, type AskQuestionInput, AskQuestionInputSchema, AwkInputSchema, type AwkToolInput, type EditFileInput, EditFileInputSchema, type ExtractDocumentInput, ExtractDocumentInputSchema, type FetchWebContentInput, FetchWebContentInputSchema, GrepInputSchema, type GrepToolInput, type ReadFileRequest, ReadFileRequestSchema, type ReadFilesInput, ReadFilesInputSchema, type RunCommandsInput, RunCommandsInputSchema, type SearchCodebaseInput, SearchCodebaseInputSchema, SedInputSchema, type SedToolInput, type SkillsInput, SkillsInputSchema, type StructuredCommandInput, StructuredCommandInputSchema, type SubmitInput, SubmitInputSchema, type WebFetchRequest, WebFetchRequestSchema, } from "./schemas";
 export { buildTaskProgressReminder, buildTaskProgressState, DEFAULT_TASK_PROGRESS_REMINDER_INTERVAL, parseTaskProgress, readTaskProgress, TASK_PROGRESS_PARAM, TASK_PROGRESS_PARAM_DESCRIPTION, type TaskProgressItem, type TaskProgressState, TaskProgressTracker, type TaskProgressTrackerOptions, withTaskProgressCapture, withTaskProgressParam, } from "./task-progress";
 export { TEAM_TOOL_NAMES } from "./team/team-tools";
-export type { ApplyPatchExecutor, AskQuestionExecutor, AwkExecutor, CreateDefaultToolsOptions, DefaultToolName, DefaultToolsConfig, EditorExecutor, FileReadExecutor, GrepExecutor, SearchExecutor, SedExecutor, ShellExecutionOptions, ShellExecutor, SkillsExecutor, SkillsExecutorSkillMetadata, SkillsExecutorWithMetadata, ToolExecutors, ToolOperationResult, VerifySubmitExecutor, WebFetchExecutor, } from "./types";
+export type { ApplyPatchExecutor, AskQuestionExecutor, AwkExecutor, CreateDefaultToolsOptions, DefaultToolName, DefaultToolsConfig, EditorExecutor, FileReadExecutor, GrepExecutor, SearchExecutor, SedExecutor, SemanticSearchExecutor, ShellExecutionOptions, ShellExecutor, SkillsExecutor, SkillsExecutorSkillMetadata, SkillsExecutorWithMetadata, ToolExecutors, ToolOperationResult, VerifySubmitExecutor, WebFetchExecutor, } from "./types";
 import { type AgentTool } from "@cerebriline/shared";
 import { type DefaultExecutorsOptions } from "./executors/index";
 import type { CreateDefaultToolsOptions, ToolExecutors } from "./types";

package/dist/extensions/tools/library-tools.d.ts

@@ -26,6 +26,20 @@ export interface LibraryScrapeConfig extends ScrapeEndpoint {
     maxPages: number;
     /** How many links deep a crawl follows. */
     maxDepth: number;
+    /**
+     * `false`: every task is offered the general `web_scrape`
+     * (`web-scrape-tool.ts`), which the librarian then uses too. Otherwise
+     * scraping is the librarian's only.
+     */
+    librarianOnly?: boolean;
+    /**
+     * What a site crawl may fetch besides its pages: how many files, how
+     * large one may be and how much in all, in megabytes. Absent, the
+     * defaults of `ScrapeSettings`.
+     */
+    maxFiles?: number;
+    maxFileMb?: number;
+    maxTotalMb?: number;
 }
 export interface LibraryToolsConfig {
     settings: LibrarySettings;

package/dist/extensions/tools/schemas.d.ts

@@ -775,16 +775,7 @@ export type AskQuestionInput = z.infer<typeof AskQuestionInputSchema>;
  * Input for the submit and exit tool
  */
 export type SubmitInput = z.infer<typeof SubmitInputSchema>;
-/**
- * What `grep`, `sed` and `awk` accept, as against what they advertise.
- *
- * The advertised shape keeps `array`, because that is the shape that batches.
- * These wrappers only widen the door: the one field each of them gets wrong is
- * the file list, and it is always wrong the same way. Every other field is the
- * canonical schema's own, so the parsed output is the canonical type already --
- * no pipe back through it, which `z.coerce` fields make ill-typed anyway.
- */
-export declare const GrepInputUnionSchema: z.ZodObject<{
+export declare const GrepInputUnionSchema: z.ZodPreprocess<z.ZodObject<{
     pattern: z.ZodString;
     ignore_case: z.ZodOptional<z.ZodBoolean>;
     invert: z.ZodOptional<z.ZodBoolean>;
@@ -797,7 +788,7 @@ export declare const GrepInputUnionSchema: z.ZodObject<{
     context: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     max_count: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     paths: z.ZodOptional<z.ZodUnion<readonly [z.ZodArray<z.ZodString>, z.ZodPipe<z.ZodString, z.ZodTransform<string[], string>>]>>;
-}, z.core.$strip>;
+}, z.core.$strip>>;
 export declare const SedInputUnionSchema: z.ZodObject<{
     script: z.ZodString;
     in_place: z.ZodOptional<z.ZodBoolean>;

package/dist/extensions/tools/types.d.ts

@@ -51,6 +51,27 @@ export interface SearchQueryOptions {
     maxPerFile?: number;
 }
 export type SearchExecutor = (query: string, cwd: string, context: AgentToolContext, options?: SearchQueryOptions) => Promise<string>;
+/**
+ * Search of the workspace's code by meaning, for `search_codebase`'s
+ * `mode: "semantic"`. A host supplies it only where the workspace is indexed:
+ * its presence is what adds the mode to the tool.
+ */
+export interface SemanticSearchExecutor {
+    /**
+     * Asked once, when a session's tools are built: whether this folder is
+     * indexed. One host serves many workspaces, so the answer is per folder.
+     */
+    available(cwd: string): boolean;
+    search(query: string, cwd: string, context: AgentToolContext, options?: SemanticSearchOptions): Promise<string>;
+}
+export interface SemanticSearchOptions {
+    /**
+     * Read a file of the folder as the caller sees it, by absolute path. A
+     * delegated agent works on its own copy: the lines a passage is on are the
+     * lines of that copy, not of the lead's file.
+     */
+    readFile?: (absolutePath: string) => Promise<string | undefined>;
+}
 /**
  * Per-call execution options for a shell executor.
  */
@@ -209,6 +230,8 @@ export interface ToolExecutors {
     readFile?: FileReadExecutor;
     /** Codebase search implementation */
     search?: SearchExecutor;
+    /** Code search by meaning; adds `mode: "semantic"` to search_codebase */
+    semanticSearch?: SemanticSearchExecutor;
     /** Shell command execution implementation */
     bash?: ShellExecutor;
     /** Web content fetching implementation */

package/dist/extensions/tools/web-scrape-tool.d.ts

@@ -0,0 +1,49 @@
+/**
+ * `web_scrape`: the web through a Firecrawl endpoint.
+ *
+ * Two shapes of one tool. The librarian's looks at the web to choose what a
+ * book is made from: search, map, read. The general one is offered to any
+ * task when the user turns "Only for the librarian" off, and adds what a task
+ * outside the Library needs: a crawl that saves a site into the workspace.
+ * By default that is the site itself -- every page as it was served and as
+ * the browser rendered it, with the stylesheets, scripts, pictures and fonts
+ * it uses -- beside a markdown reading of each page; `content: "text"` keeps
+ * the markdown alone, for notes.
+ *
+ * The lead's only: it is not in `DELEGATED_HOST_TOOLS`, so a delegated agent,
+ * whose writes go through its overlay, is never handed a tool that writes to
+ * the workspace directly.
+ */
+import { type AgentTool } from "@cerebriline/shared";
+import type { LibraryScrapeConfig } from "./library-tools";
+export { cleanLink, linkCensus, pageFileName } from "./web-site-crawl";
+export declare const WEB_SCRAPE_TOOL_NAME = "web_scrape";
+export declare const NO_SCRAPER = "Web scraping is not set up for this session. The user sets the endpoint under Settings > Features and allows it in the API configuration; do not call this again in this task.";
+export interface WebScrapeToolOptions {
+    /** The folder saved pages are written under. */
+    cwd: string;
+    /** The endpoint and its limits, read on every call. Undefined: not set up. */
+    getScrape: () => LibraryScrapeConfig | undefined;
+    /** What stops the tool before the endpoint is asked, said to the model. */
+    unavailable?: () => string | undefined;
+    /** The general tool: `crawl`, and `save_to`. @default false, the librarian's */
+    general?: boolean;
+    onError?: (message: string, error: unknown) => void;
+    /**
+     * A text file the tool wrote, by absolute path. The host forgets what it
+     * knew of the file, so a later read is not told that something outside the
+     * session changed it.
+     */
+    onWrote?: (file: string) => void;
+}
+/** `target` under `cwd`, or nothing when it would leave the folder. */
+export declare function workspacePath(cwd: string, target: string): string | undefined;
+export declare function createWebScrapeTool(options: WebScrapeToolOptions): AgentTool;
+/**
+ * The general tool, when scraping is set up, allowed, and not kept for the
+ * librarian. Decided at session start, like every other tool list; the
+ * endpoint itself is read again on every call.
+ */
+export declare function createWebScrapeTools(options: Omit<WebScrapeToolOptions, "general"> & {
+    log?: (message: string) => void;
+}): AgentTool[];

package/dist/extensions/tools/web-site-crawl.d.ts

@@ -0,0 +1,96 @@
+/**
+ * A site crawled into a folder of the workspace, and continued later.
+ *
+ * The folder keeps a record of the crawl (`.scrape.json`): the pages read
+ * and where each was saved, the files fetched, the pages that are linked and
+ * were not read, and the files a limit left out. A later crawl into the same
+ * folder with `resume` reads only what is missing -- the pages from that
+ * list, one by one through the endpoint, and the files not yet on disk --
+ * and nothing that is already there.
+ *
+ * Every limit that stopped something is said at the top of the result, as an
+ * alert the model is told to pass on: the user set those limits, and only
+ * the user can raise them.
+ */
+import { type ScrapedPage } from "../../services/retrieval/firecrawl";
+import type { LibraryScrapeConfig } from "./library-tools";
+export declare function plural(count: number, one: string, many?: string): string;
+/** An http(s) address without its fragment, or nothing. */
+export declare function cleanLink(link: string): string | undefined;
+/**
+ * A page's file under the crawl's folder: its host and path, with what a
+ * file name cannot hold replaced, so two pages of a site never collide and
+ * the folder reads like the site.
+ */
+export declare function pageFileName(url: string, taken: Set<string>): string;
+export declare function pageFile(page: ScrapedPage): string;
+/**
+ * Where the links of the pages read lead: pages of the same site that were
+ * not read, and other sites. It is what tells "the site has no more pages"
+ * from "the crawl stopped short" -- the two read the same from a page count.
+ */
+export declare function linkCensus(pages: readonly ScrapedPage[], start: string): {
+    unread: string[];
+    elsewhere: string[];
+};
+export declare function sitesLine(hosts: readonly string[], most?: number): string;
+/** The record a crawl leaves in its folder. */
+export declare const SITE_RECORD_FILE = ".scrape.json";
+export interface SiteRecord {
+    version: 1;
+    /** The address the crawl started from. */
+    url: string;
+    content: "site" | "text";
+    wholeSite: boolean;
+    include: string[];
+    exclude: string[];
+    /** Pages read, by address. */
+    pages: Record<string, {
+        title?: string;
+        markdown: string;
+        characters: number;
+        served?: string;
+        rendered?: string;
+    }>;
+    /** Files fetched, by address. */
+    assets: Record<string, {
+        file: string;
+        bytes: number;
+    }>;
+    /** Pages of the site that are linked and were not read. */
+    unread: string[];
+    /** Files a limit left out. */
+    assetsLeft: string[];
+    /** Files that could not be fetched, with the reason. */
+    failed: Record<string, string>;
+    /** Pages that could not be read, with the reason. */
+    failedPages: Record<string, string>;
+}
+export interface CrawlToFolderRequest {
+    scrape: LibraryScrapeConfig;
+    url: string;
+    /** The folder, absolute. */
+    target: string;
+    /** The folder as the model named it, for the result. */
+    folder: string;
+    limit?: number;
+    depth?: number;
+    include: string[];
+    exclude: string[];
+    wholeSite?: boolean;
+    content?: "site" | "text";
+    resume: boolean;
+    signal?: AbortSignal;
+    emit?: (status: string) => void;
+    /** A text file the crawl wrote or rewrote, by absolute path. */
+    wrote?: (file: string) => void;
+    /** The crawl stopped at a limit of the user's; see `reportStoppedForUser`. */
+    stopped?: (reason: string) => void;
+}
+/**
+ * Whether a link is one this crawl reads: the same site, below the starting
+ * address unless the whole site was asked for, and through the path filters.
+ */
+export declare function inCrawlScope(link: string, record: Pick<SiteRecord, "url" | "wholeSite" | "include" | "exclude">): boolean;
+/** Crawl a site into a folder, or continue the crawl that is there. */
+export declare function crawlToFolder(request: CrawlToFolderRequest): Promise<string>;