@autorag/librarian 2.5.1 → 2.5.3

package/dist/agent/agent.d.ts

@@ -16,6 +16,8 @@ import { type ScanDuplicateDocumentsDetails } from "./dupey-tool.js";
 import { type JikjiFindProviderResult } from "./jikji-find-tool.js";
 import { type SearchDocumentDiagnostic, type SearchDocumentsResponse, type SearchDocumentsStreamEvent } from "./search-documents.js";
 import { type WatcherFactory, type WatchRefreshHandle } from "./watch-refresh.js";
+import { type WebFetchToolOptions } from "./web-fetch-tool.js";
+import { type WebSearchToolOptions } from "./web-search-tool.js";
 export interface AutoRefreshOptions {
     readonly intervalMs: number;
     readonly immediate?: boolean;
@@ -114,6 +116,21 @@ export interface AutoRAGAgentOptions {
     tools?: AgentTool[];
     minSync?: Omit<MinSyncVectorMethodOptions, "root"> | false;
     jikji?: JikjiOptions | false;
+    /**
+     * Internet web tools (`web_search` + `web_fetch`), ported from oh-my-pi's
+     * provider-chain web module. Default enabled and credential-free: the
+     * chain leads with providers that need no user-issued key — model-native
+     * search reusing the agent's own model credentials (Gemini grounding,
+     * Anthropic/OpenAI/xAI web_search), the anonymous Perplexity ask
+     * endpoint, and Parallel's keyless MCP — then scraped engines with
+     * headless-browser escalation for bot challenges. A self-hosted
+     * SEARXNG_ENDPOINT is the only env-gated option. `false` disables both
+     * tools. The `fetch` sub-option tunes or disables `web_fetch` alone.
+     * Web tools are always omitted for remote P2P sessions.
+     */
+    webSearch?: (WebSearchToolOptions & {
+        fetch?: WebFetchToolOptions | false;
+    }) | false;
     autoRefresh?: AutoRefreshOptions;
     parserOptions?: DefaultParserRegistryOptions;
     dupey?: DupeyCliOptions | false;
@@ -150,6 +167,8 @@ export type AutoRAGJikjiPrepareResult = {
 export declare class AutoRAGAgent {
     private readonly innerAgent;
     private readonly tools;
+    /** Static SEARCH_TOOLS plus the generated per-datasource tool names. */
+    private readonly searchToolNames;
     private readonly configuredModel;
     private readonly apiKey;
     private readonly providerApiKeys;
@@ -162,6 +181,8 @@ export declare class AutoRAGAgent {
     private readonly sessions;
     private activeRun;
     private resultCapture;
+    /** This agent's model credential for model-native web search (per instance, never shared). */
+    private modelNativeSearchAuth;
     private retrievalTrace;
     private preliminaryCallback;
     /** Per-phase thinking levels; undefined marks the legacy single-phase flow. */
@@ -200,14 +221,12 @@ export declare class AutoRAGAgent {
     readonly remoteSession: boolean;
     private activeRetrievalOptions;
     private searchToolCallCount;
-    private readonly sourceSearchCallCounts;
     constructor(options: AutoRAGAgentOptions);
     scanDuplicateDocuments(): Promise<ScanDuplicateDocumentsDetails>;
     private withMemoryContext;
     private resolveSessionModel;
     private createSearchSession;
     private configureSearchSession;
-    private withPerSourceSearchLimit;
     private recordSearchToolEvent;
     private currentSystemPromptConfig;
     subscribe(listener: Parameters<Agent["subscribe"]>[0]): () => void;
@@ -242,6 +261,13 @@ export declare class AutoRAGAgent {
      * model-visible; unauthorized skills are omitted entirely (default-deny).
      */
     private buildAuthorizedDatasourceSkills;
+    /**
+     * Per-connection tool specs for the generated `search_datasource_<id>`
+     * tools, built from the same trusted config and access context as
+     * {@link buildAuthorizedDatasourceSkills}. One spec per authorized
+     * datasource skill; duplicate ids collapse to the first registration.
+     */
+    private singleDatasourceToolSpecs;
     /**
      * Resolve an authorized datasource agent skill by model-visible name for the
      * `load_datasource_skill` tool. Returns `undefined` for unknown or
@@ -368,7 +394,18 @@ export declare class AutoRAGAgent {
         results: RetrievalResult[];
         diagnostics: RetrievalDiagnostic[];
     }>;
-    searchDatasourceDocuments(query: string, options?: {
+    /**
+     * Search one datasource connection only. Only the target connection's
+     * retrieval methods are registered with the retriever, so no other
+     * datasource CLI is spawned at all and only that connection's hits are
+     * returned. Access is still gated by the trusted datasource context: an
+     * unknown or unauthorized `datasourceId` yields an empty result set.
+     *
+     * This backs the generated `search_datasource_<id>` tools; every authorized
+     * connection has one. Cross-datasource fan-out is
+     * {@link searchAllDocuments}, which spans every retrieval method.
+     */
+    searchSingleDatasourceDocuments(datasourceId: string, query: string, options?: {
         readonly topK?: number;
         readonly scope?: string;
     }): Promise<{

package/dist/agent/index.d.ts

@@ -6,7 +6,7 @@ export type { AutoRAGEmittedResult, AutoRAGMappingEntry, AutoRAGResultsDetails,
 export { createEmitResultsTool, EMIT_AUTORAG_RESULTS_TOOL_NAME, emitResultsSchema, } from "./emit-results-tool.js";
 export { createJikjiFindTool, JIKJI_FIND_TOOL_NAME, type JikjiFindDetails, type JikjiFindPerRootPolicy, type JikjiFindProvider, type JikjiFindProviderResult, type MergedJikjiPolicy, } from "./jikji-find-tool.js";
 export { createSearchAllDocumentsTool, SEARCH_ALL_DOCUMENTS_TOOL_NAME, type SearchAllDocumentsDetails, type SearchAllDocumentsProvider, type SearchAllDocumentsResult, } from "./search-all-tool.js";
-export { createSearchDatasourceDocumentsTool, type DatasourceSearchProvider, SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME, type SearchDatasourceDocumentsDetails, } from "./search-datasource-tool.js";
 export type { SearchDocumentDiagnostic, SearchDocumentDiagnosticCode, SearchDocumentDiagnosticSeverity, SearchDocumentEvidence, SearchDocumentResult, SearchDocumentsResponse, SearchDocumentsStreamEvent, SearchDocumentWarning, } from "./search-documents.js";
 export { createSearchMinSyncDocumentsTool, SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, type SearchMinSyncDocumentsDetails, } from "./search-minsync-tool.js";
+export { createSingleDatasourceSearchTools, formatDatasourceResults, SEARCH_SINGLE_DATASOURCE_TOOL_PREFIX, type SearchSingleDatasourceDetails, type SingleDatasourceSearchProvider, type SingleDatasourceToolSpec, singleDatasourceToolName, } from "./search-single-datasource-tool.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./system-prompt.js";

package/dist/agent/search-single-datasource-tool.d.ts

@@ -0,0 +1,63 @@
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { Type } from "typebox";
+import type { RetrievalDiagnostic, RetrievalResult } from "../retrieval/types.js";
+import { type SearchDocumentRetrievalTraceResult } from "./search-documents.js";
+/**
+ * Per-datasource search tools.
+ *
+ * One tool is generated per authorized datasource connection (e.g.
+ * `search_datasource_discord`, or `search_datasource_kakao_work` for a
+ * `kakao-work` account alias). Its execution registers only that connection's
+ * retrieval methods with the retriever, so no other datasource CLI is spawned
+ * and the result set carries only that connection's hits.
+ *
+ * These generated tools are the only model-facing datasource retrieval
+ * surface. Cross-datasource fan-out belongs to `search_all_documents`, which
+ * already spans every configured retrieval method — including datasources — so
+ * a datasource-only fan-out tool would be redundant. Every authorized
+ * connection must therefore appear here; a datasource with no generated tool
+ * would become unreachable except through the full fan-out.
+ *
+ * Tools are generated from the configured, access-authorized datasource skills
+ * at agent construction, so a datasource that is disabled in config or denied
+ * by the trusted access context never appears in the tool list.
+ */
+export declare const SEARCH_SINGLE_DATASOURCE_TOOL_PREFIX = "search_datasource_";
+/** Model-visible tool name for one datasource connection. */
+export declare const singleDatasourceToolName: (datasourceId: string) => string;
+/** Static per-connection descriptor used to generate one tool each. */
+export interface SingleDatasourceToolSpec {
+    /** Trusted datasource id (alias-aware, e.g. `kakao-work`). */
+    readonly datasourceId: string;
+    /** Operator/descriptor-authored context shown in the tool description. */
+    readonly description: string;
+    /** Authorized instance roots (e.g. `/kakao/personal`) for the description. */
+    readonly instanceScopes: readonly string[];
+}
+export interface SingleDatasourceSearchProvider {
+    searchSingleDatasourceDocuments(datasourceId: string, query: string, options?: {
+        readonly topK?: number;
+        readonly scope?: string;
+    }): Promise<{
+        readonly results: readonly RetrievalResult[];
+        readonly diagnostics: readonly RetrievalDiagnostic[];
+    }>;
+}
+export interface SearchSingleDatasourceDetails {
+    readonly method: "datasource";
+    readonly datasource: string;
+    readonly resultCount: number;
+    readonly sources: readonly string[];
+    readonly diagnostics: readonly RetrievalDiagnostic[];
+    /** Top candidates in traceable shape (additive; used for the run's retrieval trace). */
+    readonly results?: readonly SearchDocumentRetrievalTraceResult[];
+}
+export declare function createSingleDatasourceSearchTools(provider: SingleDatasourceSearchProvider, specs: readonly SingleDatasourceToolSpec[]): AgentTool<typeof searchSingleDatasourceSchema, SearchSingleDatasourceDetails>[];
+declare const searchSingleDatasourceSchema: Type.TObject<{
+    query: Type.TString;
+    topK: Type.TOptional<Type.TInteger>;
+    scope: Type.TOptional<Type.TString>;
+}>;
+/** Shared rendering for datasource search tool results (fan-out removed). */
+export declare function formatDatasourceResults(results: readonly RetrievalResult[], diagnostics: readonly RetrievalDiagnostic[]): string;
+export {};

package/dist/agent/web-fetch-tool.d.ts

@@ -0,0 +1,32 @@
+/**
+ * `web_fetch` agent tool — read a public web page as text/markdown.
+ *
+ * LLM-facing wrapper around the oh-my-pi-style render pipeline in
+ * `src/web/fetch/`: content-negotiation (markdown alternates, `.md` suffix,
+ * llms.txt), feed/JSON/text handling, and HTML rendered to markdown through
+ * the reader-backend chain. Output is head-truncated like oh-my-pi's
+ * read-url integration so one page cannot flood the context.
+ */
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { Type } from "typebox";
+export declare const WEB_FETCH_TOOL_NAME = "web_fetch";
+declare const webFetchSchema: Type.TObject<{
+    url: Type.TString;
+    raw: Type.TOptional<Type.TBoolean>;
+}>;
+export interface WebFetchToolOptions {
+    /** Total fetch/render timeout in seconds (default 30). */
+    readonly timeoutSeconds?: number;
+}
+export interface WebFetchToolDetails {
+    readonly method: "web_fetch";
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly contentType: string;
+    readonly fetchMethod: string;
+    readonly truncated: boolean;
+    readonly available: boolean;
+    readonly error?: string;
+}
+export declare function createWebFetchTool(options?: WebFetchToolOptions): AgentTool<typeof webFetchSchema, WebFetchToolDetails>;
+export {};

package/dist/agent/web-search-tool.d.ts

@@ -0,0 +1,48 @@
+/**
+ * `web_search` agent tool — internet web search for the librarian agent.
+ *
+ * LLM-facing wrapper around the oh-my-pi-style provider chain in
+ * `src/web/search/`: credential-free end to end (model-native search on the
+ * agent's own model credential, the anonymous Perplexity ask endpoint,
+ * Parallel's keyless MCP, then the scraped engines), where quota/auth/bot
+ * failures automatically fall back down the chain. The model only supplies
+ * `query` plus optional hints; provider selection and credentials are never
+ * exposed through tool arguments.
+ */
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { Type } from "typebox";
+import type { ModelNativeSearchAuth } from "../web/search/model-auth.js";
+import type { SearchProviderId } from "../web/search/types.js";
+export declare const WEB_SEARCH_TOOL_NAME = "web_search";
+declare const webSearchSchema: Type.TObject<{
+    query: Type.TString;
+    recency: Type.TOptional<Type.TUnion<[Type.TLiteral<"day">, Type.TLiteral<"week">, Type.TLiteral<"month">, Type.TLiteral<"year">]>>;
+    limit: Type.TOptional<Type.TInteger>;
+    num_search_results: Type.TOptional<Type.TInteger>;
+}>;
+export interface WebSearchToolOptions {
+    /** Force one provider (default: walk the configured chain). */
+    readonly provider?: SearchProviderId;
+    /** Prioritize these providers; unlisted providers keep their built-in relative order. */
+    readonly order?: readonly SearchProviderId[];
+    /** Providers never used by web search, including fallbacks. */
+    readonly exclude?: readonly SearchProviderId[];
+    /** Per-provider transport hard timeout in seconds (default 60, max 300). */
+    readonly timeoutSeconds?: number;
+    /**
+     * Agent model credential for the model-native providers, read per call so
+     * the tool follows its own agent's session model and never another
+     * agent's. Returning undefined leaves the env-key fallback in charge.
+     */
+    readonly modelAuth?: () => ModelNativeSearchAuth | undefined;
+}
+export interface WebSearchToolDetails {
+    readonly method: "web_search";
+    readonly provider: string;
+    readonly resultCount: number;
+    readonly sources: readonly string[];
+    readonly available: boolean;
+    readonly error?: string;
+}
+export declare function createWebSearchTool(options?: WebSearchToolOptions): AgentTool<typeof webSearchSchema, WebSearchToolDetails>;
+export {};

package/dist/cli/commands/lite-retrieve.d.ts

@@ -8,8 +8,23 @@ export interface LiteRetrieveEnvelope {
      */
     readonly stale: boolean;
     readonly results: readonly LiteRetrieveResultItem[];
+    /**
+     * Retrieval surfaces that did not run for this query. A consumer that only
+     * reads `results` can check `unsearched.length > 0` to learn the answer is
+     * partial — for example local MinSync files skipped while an index sync holds
+     * the workspace lock, leaving only datasource hits.
+     */
+    readonly unsearched: readonly LiteRetrieveUnsearched[];
     readonly diagnostics: readonly LiteRetrieveDiagnostic[];
 }
+export interface LiteRetrieveUnsearched {
+    /** Surface label: "minsync" for local files, or the datasource id. */
+    readonly surface: string;
+    /** Retrieval method names on this surface that did not run. */
+    readonly methods: readonly string[];
+    /** The underlying failure verbatim, including CLI stderr and exit status. */
+    readonly reason: string;
+}
 export interface LiteRetrieveResultItem {
     readonly number: number;
     readonly source: string;

package/dist/cli/commands/search.d.ts

@@ -26,6 +26,12 @@ export declare function classifySearchHealthHint(error: unknown): SearchHealthHi
 export interface SearchDeps {
     agentFactory?: (opts: AutoRAGAgentOptions) => Pick<AutoRAGAgent, "searchDocumentsStream">;
     modelResolver?: (config: CliConfig) => ResolvedAgentModel;
+    /**
+     * Stops the on-demand embedding gateway so the process can exit. Without
+     * this the semantic MinSync path's loopback gateway keeps the event loop
+     * alive after the search completes and the CLI never exits.
+     */
+    stopRuntime?: () => Promise<void>;
 }
 /**
  * Run the `autorag search` command. Returns exit code 0 on success, 2 for

package/dist/cli/config.d.ts

@@ -31,6 +31,25 @@ export interface RawIndexingMethods {
 export interface NormalizedIndexingConfig {
     minSync: MinSyncMethodConfig;
 }
+/**
+ * Web tools (`web_search` + `web_fetch`) config. Secrets never appear here:
+ * the shipped chain is credential-free (model-native providers reuse the
+ * agent's resolved model credentials; perplexity/parallel need nothing), and
+ * the only env-gated option is a self-hosted SEARXNG_ENDPOINT.
+ */
+export interface WebSearchCliConfig {
+    enabled?: boolean;
+    provider?: string;
+    order?: string[];
+    exclude?: string[];
+    /** Per-provider transport hard timeout in seconds (1-300). */
+    timeoutSeconds?: number;
+    fetch?: {
+        enabled?: boolean;
+        /** Total page fetch/render timeout in seconds (1-300). */
+        timeoutSeconds?: number;
+    } | false;
+}
 export interface P2pConfig {
     enabled?: boolean;
     port?: number;
@@ -91,6 +110,7 @@ export interface CliConfig {
     model?: AgentModelConfig;
     minSync?: MinSyncMethodConfig;
     jikji?: Record<string, unknown> | false;
+    webSearch?: WebSearchCliConfig;
     parserOptions?: Record<string, unknown>;
     dupey?: {
         enabled?: boolean;