@autorag/librarian 2.5.0 → 2.5.2

package/dist/agent/agent.d.ts

@@ -16,6 +16,8 @@ import { type ScanDuplicateDocumentsDetails } from "./dupey-tool.js";
 import { type JikjiFindProviderResult } from "./jikji-find-tool.js";
 import { type SearchDocumentDiagnostic, type SearchDocumentsResponse, type SearchDocumentsStreamEvent } from "./search-documents.js";
 import { type WatcherFactory, type WatchRefreshHandle } from "./watch-refresh.js";
+import { type WebFetchToolOptions } from "./web-fetch-tool.js";
+import { type WebSearchToolOptions } from "./web-search-tool.js";
 export interface AutoRefreshOptions {
     readonly intervalMs: number;
     readonly immediate?: boolean;
@@ -30,6 +32,9 @@ export interface AutoRAGMinSyncRefreshResult {
     readonly ok: boolean;
     readonly synced: number;
     readonly reason?: string;
+    /** Count of parsed documents excluded from the MinSync index by file name. */
+    readonly stagingExcludedCount?: number;
+    readonly diagnostics?: readonly SearchDocumentDiagnostic[];
 }
 export interface AutoRAGRefreshResult extends Omit<ParsedMirrorSyncResult, "diagnostics"> {
     readonly diagnostics: readonly SearchDocumentDiagnostic[];
@@ -111,6 +116,21 @@ export interface AutoRAGAgentOptions {
     tools?: AgentTool[];
     minSync?: Omit<MinSyncVectorMethodOptions, "root"> | false;
     jikji?: JikjiOptions | false;
+    /**
+     * Internet web tools (`web_search` + `web_fetch`), ported from oh-my-pi's
+     * provider-chain web module. Default enabled and credential-free: the
+     * chain leads with providers that need no user-issued key — model-native
+     * search reusing the agent's own model credentials (Gemini grounding,
+     * Anthropic/OpenAI/xAI web_search), the anonymous Perplexity ask
+     * endpoint, and Parallel's keyless MCP — then scraped engines with
+     * headless-browser escalation for bot challenges. A self-hosted
+     * SEARXNG_ENDPOINT is the only env-gated option. `false` disables both
+     * tools. The `fetch` sub-option tunes or disables `web_fetch` alone.
+     * Web tools are always omitted for remote P2P sessions.
+     */
+    webSearch?: (WebSearchToolOptions & {
+        fetch?: WebFetchToolOptions | false;
+    }) | false;
     autoRefresh?: AutoRefreshOptions;
     parserOptions?: DefaultParserRegistryOptions;
     dupey?: DupeyCliOptions | false;
@@ -147,6 +167,8 @@ export type AutoRAGJikjiPrepareResult = {
 export declare class AutoRAGAgent {
     private readonly innerAgent;
     private readonly tools;
+    /** Static SEARCH_TOOLS plus the generated per-datasource tool names. */
+    private readonly searchToolNames;
     private readonly configuredModel;
     private readonly apiKey;
     private readonly providerApiKeys;
@@ -159,6 +181,8 @@ export declare class AutoRAGAgent {
     private readonly sessions;
     private activeRun;
     private resultCapture;
+    /** This agent's model credential for model-native web search (per instance, never shared). */
+    private modelNativeSearchAuth;
     private retrievalTrace;
     private preliminaryCallback;
     /** Per-phase thinking levels; undefined marks the legacy single-phase flow. */
@@ -197,14 +221,12 @@ export declare class AutoRAGAgent {
     readonly remoteSession: boolean;
     private activeRetrievalOptions;
     private searchToolCallCount;
-    private readonly sourceSearchCallCounts;
     constructor(options: AutoRAGAgentOptions);
     scanDuplicateDocuments(): Promise<ScanDuplicateDocumentsDetails>;
     private withMemoryContext;
     private resolveSessionModel;
     private createSearchSession;
     private configureSearchSession;
-    private withPerSourceSearchLimit;
     private recordSearchToolEvent;
     private currentSystemPromptConfig;
     subscribe(listener: Parameters<Agent["subscribe"]>[0]): () => void;
@@ -239,6 +261,13 @@ export declare class AutoRAGAgent {
      * model-visible; unauthorized skills are omitted entirely (default-deny).
      */
     private buildAuthorizedDatasourceSkills;
+    /**
+     * Per-connection tool specs for the generated `search_datasource_<id>`
+     * tools, built from the same trusted config and access context as
+     * {@link buildAuthorizedDatasourceSkills}. One spec per authorized
+     * datasource skill; duplicate ids collapse to the first registration.
+     */
+    private singleDatasourceToolSpecs;
     /**
      * Resolve an authorized datasource agent skill by model-visible name for the
      * `load_datasource_skill` tool. Returns `undefined` for unknown or
@@ -273,6 +302,11 @@ export declare class AutoRAGAgent {
     /**
      * Path-opaque snapshot of corpus freshness and the last refresh outcome. Runs
      * a cheap parse-free staleness scan (stat only); never parses in this path.
+     *
+     * Freshness is read from disk, not from this instance's history: a separate CLI
+     * process (for example `autorag status` or `autorag lite status`) reports the
+     * corpus as current when the last refresh left parsed mirrors behind and no
+     * source has changed since.
      */
     getRefreshStatus(): Promise<AutoRAGRefreshStatus>;
     /**
@@ -367,6 +401,22 @@ export declare class AutoRAGAgent {
         results: RetrievalResult[];
         diagnostics: RetrievalDiagnostic[];
     }>;
+    /**
+     * Search one datasource connection only. Unlike
+     * {@link searchDatasourceDocuments}, which fans out to every authorized
+     * datasource and filters afterwards, this registers only the target
+     * connection's retrieval methods with the retriever, so no other
+     * datasource CLI is spawned at all. Access is still gated by the trusted
+     * datasource context: an unknown or unauthorized `datasourceId` yields an
+     * empty result set.
+     */
+    searchSingleDatasourceDocuments(datasourceId: string, query: string, options?: {
+        readonly topK?: number;
+        readonly scope?: string;
+    }): Promise<{
+        results: RetrievalResult[];
+        diagnostics: RetrievalDiagnostic[];
+    }>;
     /** The retrieval method registry (posix, MinSync, and datasource methods). */
     getMethodRegistry(): RetrievalMethodRegistry;
     /**

package/dist/agent/index.d.ts

@@ -9,4 +9,5 @@ export { createSearchAllDocumentsTool, SEARCH_ALL_DOCUMENTS_TOOL_NAME, type Sear
 export { createSearchDatasourceDocumentsTool, type DatasourceSearchProvider, SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME, type SearchDatasourceDocumentsDetails, } from "./search-datasource-tool.js";
 export type { SearchDocumentDiagnostic, SearchDocumentDiagnosticCode, SearchDocumentDiagnosticSeverity, SearchDocumentEvidence, SearchDocumentResult, SearchDocumentsResponse, SearchDocumentsStreamEvent, SearchDocumentWarning, } from "./search-documents.js";
 export { createSearchMinSyncDocumentsTool, SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, type SearchMinSyncDocumentsDetails, } from "./search-minsync-tool.js";
+export { createSingleDatasourceSearchTools, SEARCH_SINGLE_DATASOURCE_TOOL_PREFIX, type SearchSingleDatasourceDetails, type SingleDatasourceSearchProvider, type SingleDatasourceToolSpec, singleDatasourceToolName, } from "./search-single-datasource-tool.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./system-prompt.js";

package/dist/agent/search-datasource-tool.d.ts

@@ -26,4 +26,5 @@ export interface DatasourceSearchProvider {
     }>;
 }
 export declare function createSearchDatasourceDocumentsTool(provider: DatasourceSearchProvider): AgentTool<typeof searchDatasourceSchema, SearchDatasourceDocumentsDetails>;
+export declare function formatDatasourceResults(results: readonly RetrievalResult[], diagnostics: readonly RetrievalDiagnostic[]): string;
 export {};

package/dist/agent/search-documents.d.ts

@@ -10,7 +10,7 @@ export type SearchDocumentDiagnosticSeverity = "info" | "warning" | "error";
  * (#21/#22) does not require another source-breaking type change; codes not yet
  * emitted are still valid members.
  */
-export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-unavailable" | "parser-skipped" | "parser-failed" | "pdf-java-version" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "pdf-hybrid-unavailable" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "missing-final-emit";
+export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-unavailable" | "minsync-staging-excluded" | "minsync-sync-failed" | "embedder-unavailable" | "embedding-identity-mismatch" | "parser-skipped" | "parser-failed" | "pdf-java-version" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "pdf-hybrid-unavailable" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "missing-final-emit";
 export interface SearchDocumentDiagnostic {
     readonly code: SearchDocumentDiagnosticCode;
     readonly severity: SearchDocumentDiagnosticSeverity;

package/dist/agent/search-single-datasource-tool.d.ts

@@ -0,0 +1,57 @@
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { Type } from "typebox";
+import type { RetrievalDiagnostic, RetrievalResult } from "../retrieval/types.js";
+import { type SearchDocumentRetrievalTraceResult } from "./search-documents.js";
+/**
+ * Per-datasource search tools.
+ *
+ * `search_datasource_documents` always fans out to every authorized datasource:
+ * the retriever spawns every datasource CLI and only then filters results, so
+ * an agent that already knows which connection holds the answer (from memory or
+ * from the question itself) still pays for — and receives — every other
+ * datasource's hits. These generated tools fix that: one tool per authorized
+ * datasource connection (e.g. `search_datasource_discord`,
+ * `search_datasource_kakao_work` for a `kakao-work` account alias) whose
+ * execution spawns only that connection's retrieval methods.
+ *
+ * Tools are generated from the configured, access-authorized datasource skills
+ * at agent construction, so a datasource that is disabled in config or denied
+ * by the trusted access context never appears in the tool list.
+ */
+export declare const SEARCH_SINGLE_DATASOURCE_TOOL_PREFIX = "search_datasource_";
+/** Model-visible tool name for one datasource connection. */
+export declare const singleDatasourceToolName: (datasourceId: string) => string;
+/** Static per-connection descriptor used to generate one tool each. */
+export interface SingleDatasourceToolSpec {
+    /** Trusted datasource id (alias-aware, e.g. `kakao-work`). */
+    readonly datasourceId: string;
+    /** Operator/descriptor-authored context shown in the tool description. */
+    readonly description: string;
+    /** Authorized instance roots (e.g. `/kakao/personal`) for the description. */
+    readonly instanceScopes: readonly string[];
+}
+export interface SingleDatasourceSearchProvider {
+    searchSingleDatasourceDocuments(datasourceId: string, query: string, options?: {
+        readonly topK?: number;
+        readonly scope?: string;
+    }): Promise<{
+        readonly results: readonly RetrievalResult[];
+        readonly diagnostics: readonly RetrievalDiagnostic[];
+    }>;
+}
+export interface SearchSingleDatasourceDetails {
+    readonly method: "datasource";
+    readonly datasource: string;
+    readonly resultCount: number;
+    readonly sources: readonly string[];
+    readonly diagnostics: readonly RetrievalDiagnostic[];
+    /** Top candidates in traceable shape (additive; used for the run's retrieval trace). */
+    readonly results?: readonly SearchDocumentRetrievalTraceResult[];
+}
+export declare function createSingleDatasourceSearchTools(provider: SingleDatasourceSearchProvider, specs: readonly SingleDatasourceToolSpec[]): AgentTool<typeof searchSingleDatasourceSchema, SearchSingleDatasourceDetails>[];
+declare const searchSingleDatasourceSchema: Type.TObject<{
+    query: Type.TString;
+    topK: Type.TOptional<Type.TInteger>;
+    scope: Type.TOptional<Type.TString>;
+}>;
+export {};

package/dist/agent/web-fetch-tool.d.ts

@@ -0,0 +1,32 @@
+/**
+ * `web_fetch` agent tool — read a public web page as text/markdown.
+ *
+ * LLM-facing wrapper around the oh-my-pi-style render pipeline in
+ * `src/web/fetch/`: content-negotiation (markdown alternates, `.md` suffix,
+ * llms.txt), feed/JSON/text handling, and HTML rendered to markdown through
+ * the reader-backend chain. Output is head-truncated like oh-my-pi's
+ * read-url integration so one page cannot flood the context.
+ */
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { Type } from "typebox";
+export declare const WEB_FETCH_TOOL_NAME = "web_fetch";
+declare const webFetchSchema: Type.TObject<{
+    url: Type.TString;
+    raw: Type.TOptional<Type.TBoolean>;
+}>;
+export interface WebFetchToolOptions {
+    /** Total fetch/render timeout in seconds (default 30). */
+    readonly timeoutSeconds?: number;
+}
+export interface WebFetchToolDetails {
+    readonly method: "web_fetch";
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly contentType: string;
+    readonly fetchMethod: string;
+    readonly truncated: boolean;
+    readonly available: boolean;
+    readonly error?: string;
+}
+export declare function createWebFetchTool(options?: WebFetchToolOptions): AgentTool<typeof webFetchSchema, WebFetchToolDetails>;
+export {};

package/dist/agent/web-search-tool.d.ts

@@ -0,0 +1,48 @@
+/**
+ * `web_search` agent tool — internet web search for the librarian agent.
+ *
+ * LLM-facing wrapper around the oh-my-pi-style provider chain in
+ * `src/web/search/`: credential-free end to end (model-native search on the
+ * agent's own model credential, the anonymous Perplexity ask endpoint,
+ * Parallel's keyless MCP, then the scraped engines), where quota/auth/bot
+ * failures automatically fall back down the chain. The model only supplies
+ * `query` plus optional hints; provider selection and credentials are never
+ * exposed through tool arguments.
+ */
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { Type } from "typebox";
+import type { ModelNativeSearchAuth } from "../web/search/model-auth.js";
+import type { SearchProviderId } from "../web/search/types.js";
+export declare const WEB_SEARCH_TOOL_NAME = "web_search";
+declare const webSearchSchema: Type.TObject<{
+    query: Type.TString;
+    recency: Type.TOptional<Type.TUnion<[Type.TLiteral<"day">, Type.TLiteral<"week">, Type.TLiteral<"month">, Type.TLiteral<"year">]>>;
+    limit: Type.TOptional<Type.TInteger>;
+    num_search_results: Type.TOptional<Type.TInteger>;
+}>;
+export interface WebSearchToolOptions {
+    /** Force one provider (default: walk the configured chain). */
+    readonly provider?: SearchProviderId;
+    /** Prioritize these providers; unlisted providers keep their built-in relative order. */
+    readonly order?: readonly SearchProviderId[];
+    /** Providers never used by web search, including fallbacks. */
+    readonly exclude?: readonly SearchProviderId[];
+    /** Per-provider transport hard timeout in seconds (default 60, max 300). */
+    readonly timeoutSeconds?: number;
+    /**
+     * Agent model credential for the model-native providers, read per call so
+     * the tool follows its own agent's session model and never another
+     * agent's. Returning undefined leaves the env-key fallback in charge.
+     */
+    readonly modelAuth?: () => ModelNativeSearchAuth | undefined;
+}
+export interface WebSearchToolDetails {
+    readonly method: "web_search";
+    readonly provider: string;
+    readonly resultCount: number;
+    readonly sources: readonly string[];
+    readonly available: boolean;
+    readonly error?: string;
+}
+export declare function createWebSearchTool(options?: WebSearchToolOptions): AgentTool<typeof webSearchSchema, WebSearchToolDetails>;
+export {};

package/dist/cli/command-help.d.ts

@@ -0,0 +1 @@
+export declare function commandUsage(command: string | undefined): string | undefined;

package/dist/cli/commands/lite-retrieve.d.ts

@@ -2,9 +2,29 @@ import type { CommandContext } from "./types.js";
 export interface LiteRetrieveEnvelope {
     readonly ok: true;
     readonly query: string;
+    /**
+     * True when the last refresh does not cover the current sources. Retrieval still
+     * runs; the caller decides whether to trust the result or refresh first.
+     */
+    readonly stale: boolean;
     readonly results: readonly LiteRetrieveResultItem[];
+    /**
+     * Retrieval surfaces that did not run for this query. A consumer that only
+     * reads `results` can check `unsearched.length > 0` to learn the answer is
+     * partial — for example local MinSync files skipped while an index sync holds
+     * the workspace lock, leaving only datasource hits.
+     */
+    readonly unsearched: readonly LiteRetrieveUnsearched[];
     readonly diagnostics: readonly LiteRetrieveDiagnostic[];
 }
+export interface LiteRetrieveUnsearched {
+    /** Surface label: "minsync" for local files, or the datasource id. */
+    readonly surface: string;
+    /** Retrieval method names on this surface that did not run. */
+    readonly methods: readonly string[];
+    /** The underlying failure verbatim, including CLI stderr and exit status. */
+    readonly reason: string;
+}
 export interface LiteRetrieveResultItem {
     readonly number: number;
     readonly source: string;
@@ -18,6 +38,10 @@ export interface LiteRetrieveDiagnostic {
     readonly severity: string;
     readonly message: string;
     readonly source?: string;
+    /** Why a source is considered stale, e.g. `mtime-and-size-changed`. */
+    readonly reason?: string;
+    /** Suggested recovery action, e.g. `refresh`. */
+    readonly action?: string;
 }
 /**
  * Run the `autorag lite retrieve` command.

package/dist/cli/commands/search.d.ts

@@ -26,6 +26,12 @@ export declare function classifySearchHealthHint(error: unknown): SearchHealthHi
 export interface SearchDeps {
     agentFactory?: (opts: AutoRAGAgentOptions) => Pick<AutoRAGAgent, "searchDocumentsStream">;
     modelResolver?: (config: CliConfig) => ResolvedAgentModel;
+    /**
+     * Stops the on-demand embedding gateway so the process can exit. Without
+     * this the semantic MinSync path's loopback gateway keeps the event loop
+     * alive after the search completes and the CLI never exits.
+     */
+    stopRuntime?: () => Promise<void>;
 }
 /**
  * Run the `autorag search` command. Returns exit code 0 on success, 2 for

package/dist/cli/config.d.ts

@@ -31,6 +31,25 @@ export interface RawIndexingMethods {
 export interface NormalizedIndexingConfig {
     minSync: MinSyncMethodConfig;
 }
+/**
+ * Web tools (`web_search` + `web_fetch`) config. Secrets never appear here:
+ * the shipped chain is credential-free (model-native providers reuse the
+ * agent's resolved model credentials; perplexity/parallel need nothing), and
+ * the only env-gated option is a self-hosted SEARXNG_ENDPOINT.
+ */
+export interface WebSearchCliConfig {
+    enabled?: boolean;
+    provider?: string;
+    order?: string[];
+    exclude?: string[];
+    /** Per-provider transport hard timeout in seconds (1-300). */
+    timeoutSeconds?: number;
+    fetch?: {
+        enabled?: boolean;
+        /** Total page fetch/render timeout in seconds (1-300). */
+        timeoutSeconds?: number;
+    } | false;
+}
 export interface P2pConfig {
     enabled?: boolean;
     port?: number;
@@ -91,6 +110,7 @@ export interface CliConfig {
     model?: AgentModelConfig;
     minSync?: MinSyncMethodConfig;
     jikji?: Record<string, unknown> | false;
+    webSearch?: WebSearchCliConfig;
     parserOptions?: Record<string, unknown>;
     dupey?: {
         enabled?: boolean;

package/dist/cli/index.d.ts

@@ -1,4 +1,7 @@
 #!/usr/bin/env node
+declare const COMMANDS: readonly ["init", "setup", "refresh", "status", "search", "feedback", "evidence", "memory", "index", "watch", "health", "duplicates", "tui", "ui", "serve", "p2p", "lite", "models", "gateway"];
+type CommandName = (typeof COMMANDS)[number];
+export type { CommandName };
 interface ParsedArgs {
     positionals: string[];
     flags: Record<string, string | boolean>;
@@ -7,4 +10,3 @@ export declare function parseArgs(argv: readonly string[]): ParsedArgs | {
     error: string;
 };
 export declare function main(argv: readonly string[]): Promise<number>;
-export {};