@autorag/librarian 2.4.0 → 2.4.1

package/README.md

@@ -16,6 +16,25 @@ AutoRAG itself is the specialized search agent, not a coordinator for other
 model roles. You configure one model, and that model owns the complete
 retrieval, reading, judgment, and curation loop.
 
+## Core values
+
+Three principles drive every design decision:
+
+1. **Never migrate your data to search it.** AutoRAG federates CLI-owned
+   stores (katok, discrawl, qmd, msgvault, rclone, …) in place. No forced
+   ingestion into a central index, no third-party server holding a copy of
+   your corpus. Results carry source-native identities
+   (`kakao:<chat>/<sender>/<chunk>`) with scope-checked access — see the
+   [competitive landscape study](docs/competitive-landscape-2026-09.md) for
+   why this is the durable differentiator.
+2. **Just works — no RAG degree required.** Install it and it works: minimal
+   configuration, no pipeline tuning, no vector-DB operations, no API keys.
+   The local embedder (EmbeddingGemma via Ollama) and MinSync auto-install
+   handle the "RAG plumbing" invisibly.
+3. **Fast by design.** One configured model owns the whole loop; retrieval
+   runs locally over MinSync CDC chunks (BM25 / vector / hybrid); interactive
+   search is optimized for low latency.
+
 ## Why AutoRAG
 
 ### The problem with search tools
@@ -205,7 +224,7 @@ Security defaults are intentionally strict:
 | Notion | `notion` | external [`notcrawl`](https://github.com/openclaw/notcrawl) CLI | local-first page/database/block archive + FTS5 search |
 | GitHub Issues/PRs | `github` | GitHub REST (token optional) | issues + PR bodies per `owner/repo`; public repos work unauthenticated |
 | Cloud drives | `cloud-drive` | **[`rclone`](https://rclone.org) CLI** | Incremental Google Drive Tier-1; OneDrive/network remotes; iCloud experimental |
-| Gmail / IMAP | `gmail` | Gmail REST v1, or **[`himalaya`](https://pimalaya.org) CLI** (`backend: "himalaya"`) | the himalaya backend indexes any IMAP/Maildir account it has configured — no OAuth plumbing |
+| Gmail | `gmail` | Gmail REST v1 | OAuth access token is referenced by environment variable name |
 | Local mail exports | `mail-export` | filesystem (`.mbox` / `.eml`) | classic `From_` splitting, mailparser-based; count-only warnings |
 | Mail archives | `mailcrawl` | external [`mailcrawl`](https://github.com/NomaDamas/mailcrawl) CLI | local email sync plus BM25, semantic, and hybrid retrieval |
 | Obsidian vault | `obsidian` | external [`qmd`](https://github.com/tobi/qmd) CLI | incremental `qmd update`, BM25 `qmd search`, semantic `qmd vsearch`; vault path via `connector.vaultPath` |
@@ -223,7 +242,7 @@ Configure them in `config.json` (CLI) or pass `datasourceSkills` programmaticall
     "slack":    { "connector": { "binaryPath": "slacrawl", "configPath": "/path/to/slacrawl.yaml", "syncSource": "primary" } },
     "notion":   { "connector": { "binaryPath": "notcrawl", "configPath": "/path/to/notcrawl.yaml" } },
     "github":   { "connector": { "repos": ["owner/repo"] } },
-    "gmail":    { "connector": { "backend": "himalaya", "account": "gmail", "folder": "INBOX" } },
+    "gmail":    { "connector": { "tokenEnv": "GMAIL_ACCESS_TOKEN", "labelIds": ["INBOX"] } },
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
     "personal-google-drive": { "type": "cloud-drive", "instanceId": "personal", "connector": { "provider": "google-drive", "remote": "personal-gdrive:", "include": ["**/*.md"] } },
     "company-onedrive": { "type": "cloud-drive", "instanceId": "work", "connector": { "provider": "onedrive", "remote": "company-onedrive:Documents" } },
@@ -253,6 +272,9 @@ in mailcrawl's native store unless the operator explicitly sets
 `connector.dataDir`; Himalaya credentials and provider configuration remain
 owned by mailcrawl. 0.1.3 and earlier fail a repeated
 `index` after a no-op sync.
+Use `mailcrawl` for all Himalaya-backed IMAP/Maildir retrieval. The retired
+`gmail` connector option `backend: "himalaya"` is no longer registered; migrate
+that configuration to an explicit `mailcrawl` datasource.
 
 Install and authenticate rclone separately (`brew install rclone && rclone
 config` on macOS), then configure the provider-neutral `cloud-drive` skill.

package/dist/agent/agent.d.ts

@@ -5,22 +5,21 @@ import type { DatasourceIndexResult, DatasourceSkill } from "../datasource/types
 import { type DupeyCliOptions } from "../dupey/index.js";
 import { type JikjiFailureReason, type JikjiOptions } from "../jikji/index.js";
 import type { ResultFeedback } from "../memory/memory.js";
-import { type MinSyncBM25MethodOptions, type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
+import { type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
 import { type ParsedMirrorSyncResult } from "../mirror/sync.js";
 import type { DefaultParserRegistryOptions } from "../parser/index.js";
-import { type BM25SyncResult } from "../retrieval/methods/bm25.js";
 import { RetrievalMethodRegistry } from "../retrieval/registry.js";
 import type { CuratedResult, RetrievalDiagnostic, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
 import { type ScanDuplicateDocumentsDetails } from "./dupey-tool.js";
 import { type JikjiFindProviderResult } from "./jikji-find-tool.js";
-import { type SearchDocumentDiagnostic, type SearchDocumentsResponse } from "./search-documents.js";
+import { type SearchDocumentDiagnostic, type SearchDocumentsResponse, type SearchDocumentsStreamEvent } from "./search-documents.js";
 import { type WatcherFactory, type WatchRefreshHandle } from "./watch-refresh.js";
 export interface AutoRefreshOptions {
     readonly intervalMs: number;
     readonly immediate?: boolean;
 }
 /** Methods that `refresh` can selectively run. Defaults to all when omitted. */
-export type RefreshMethod = "parsed" | "bm25" | "minsync" | "datasources" | "jikji";
+export type RefreshMethod = "parsed" | "minsync" | "datasources" | "jikji";
 export interface AutoRAGRefreshOptions {
     /** Restrict refresh to specific methods. Defaults to all when undefined. */
     readonly methods?: readonly RefreshMethod[];
@@ -32,12 +31,10 @@ export interface AutoRAGMinSyncRefreshResult {
 }
 export interface AutoRAGRefreshResult extends Omit<ParsedMirrorSyncResult, "diagnostics"> {
     readonly diagnostics: readonly SearchDocumentDiagnostic[];
-    readonly bm25?: BM25SyncResult;
     readonly minsync?: AutoRAGMinSyncRefreshResult;
     readonly datasources?: readonly DatasourceIndexResult[];
 }
 export interface AutoRAGRefreshComponentStatus {
-    readonly bm25?: string;
     readonly minsync?: string;
     readonly jikji?: string;
     readonly datasources?: string;
@@ -78,7 +75,6 @@ export interface AutoRAGAgentOptions {
     workspacePath?: string;
     tools?: AgentTool[];
     minSync?: Omit<MinSyncVectorMethodOptions, "root"> | false;
-    bm25?: Omit<MinSyncBM25MethodOptions, "root"> | false;
     jikji?: JikjiOptions | false;
     autoRefresh?: AutoRefreshOptions;
     parserOptions?: DefaultParserRegistryOptions;
@@ -126,6 +122,10 @@ export declare class AutoRAGAgent {
     private resultCapture;
     private autoRefreshTimer;
     private refreshing;
+    private jikjiPrepareInFlight;
+    private jikjiReady;
+    private minSyncPrepareInFlight;
+    private minSyncReady;
     private refreshState;
     private readonly searchPaths;
     private readonly configuredSearchPaths;
@@ -137,7 +137,6 @@ export declare class AutoRAGAgent {
     private readonly merger;
     private readonly datasourceFilter;
     private readonly minSyncMethod;
-    private readonly bm25Method;
     private readonly jikjiClient;
     private readonly datasourceSkills;
     private readonly datasourceAccessOptions;
@@ -151,12 +150,14 @@ export declare class AutoRAGAgent {
     private readonly searchTimeoutMs;
     private readonly maxSearchToolCalls;
     private searchToolCallCount;
+    private readonly sourceSearchCallCounts;
     constructor(options: AutoRAGAgentOptions);
     scanDuplicateDocuments(): Promise<ScanDuplicateDocumentsDetails>;
     private withMemoryContext;
     private resolveSessionModel;
     private createSearchSession;
     private configureSearchSession;
+    private withPerSourceSearchLimit;
     private recordSearchToolEvent;
     private currentSystemPromptConfig;
     subscribe(listener: Parameters<Agent["subscribe"]>[0]): () => void;
@@ -177,6 +178,13 @@ export declare class AutoRAGAgent {
     recordFeedbackByIds(usefulFeedbackIds: readonly string[], notUsefulFeedbackIds?: readonly string[]): void;
     getResultRegistry(sessionId?: string): ReadonlyMap<number, CuratedResult>;
     searchDocuments(query: string, options?: RetrievalOptions): Promise<SearchDocumentsResponse>;
+    /**
+     * Stream bounded progress updates while retaining the stable
+     * {@link searchDocuments} promise API. Progress is sourced from model text
+     * deltas, so callers can render it in a TUI, CLI, or parent agent and can
+     * stop by calling {@link abort}.
+     */
+    searchDocumentsStream(query: string, options?: RetrievalOptions): AsyncGenerator<SearchDocumentsStreamEvent, void, void>;
     private datasourceAccessContext;
     /**
      * Build the Pi agent-skill list for datasource skills authorized by the
@@ -191,9 +199,16 @@ export declare class AutoRAGAgent {
      */
     loadDatasourceSkill(name: string): Skill | undefined;
     private indexDatasources;
-    /** Path-opaque component diagnostics (e.g. BM25 readiness) for the search response. */
+    /** Path-opaque component diagnostics for the search response. */
     private collectComponentDiagnostics;
     private prefetchInitialRetrievalContext;
+    /**
+     * Prepare MinSync lazily without holding up the current search. Parsed
+     * mirrors are built first because MinSync stages and indexes those mirrors.
+     */
+    /** Prepare MinSync in the background; the returned promise is for tests. */
+    scheduleMinSyncPrepareForTest(): Promise<MinSyncSyncResult | undefined> | undefined;
+    private scheduleMinSyncPrepare;
     buildSearchPrompt(query: string, options: RetrievalOptions, initialRetrievalContext?: string): string;
     refresh(force?: boolean, opts?: AutoRAGRefreshOptions): Promise<AutoRAGRefreshResult>;
     /**
@@ -221,10 +236,15 @@ export declare class AutoRAGAgent {
      */
     private scanMirrorStaleness;
     private exactDuplicateExclusions;
-    syncBM25(minsync?: MinSyncSyncResult): Promise<BM25SyncResult | undefined>;
     syncMinSync(): Promise<MinSyncSyncResult | undefined>;
     prepareJikji(): Promise<readonly AutoRAGJikjiPrepareResult[] | undefined>;
     private executeJikjiPrepare;
+    /**
+     * Start Jikji preparation without delaying the current search turn. A
+     * subsequent turn can use the prepared index, while the child process keeps
+     * running after emit_autorag_results has terminated the agent loop.
+     */
+    private scheduleJikjiPrepare;
     private sanitizeJikjiPrepareResult;
     /**
      * Provider method for the jikji_find tool. Runs JikjiClient.find over all
@@ -283,7 +303,7 @@ export declare class AutoRAGAgent {
         results: RetrievalResult[];
         diagnostics: RetrievalDiagnostic[];
     }>;
-    /** The retrieval method registry (posix active; vector/bm25/hybrid pluggable). */
+    /** The retrieval method registry (posix, MinSync, and datasource methods). */
     getMethodRegistry(): RetrievalMethodRegistry;
     getSystemPrompt(): string;
     private rerankWithMemory;

package/dist/agent/index.d.ts

@@ -7,6 +7,6 @@ export { createEmitResultsTool, EMIT_AUTORAG_RESULTS_TOOL_NAME } from "./emit-re
 export { createJikjiFindTool, JIKJI_FIND_TOOL_NAME, type JikjiFindDetails, type JikjiFindPerRootPolicy, type JikjiFindProvider, type JikjiFindProviderResult, type MergedJikjiPolicy, } from "./jikji-find-tool.js";
 export { createSearchAllDocumentsTool, SEARCH_ALL_DOCUMENTS_TOOL_NAME, type SearchAllDocumentsDetails, type SearchAllDocumentsProvider, type SearchAllDocumentsResult, } from "./search-all-tool.js";
 export { createSearchDatasourceDocumentsTool, type DatasourceSearchProvider, SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME, type SearchDatasourceDocumentsDetails, } from "./search-datasource-tool.js";
-export type { SearchDocumentDiagnostic, SearchDocumentDiagnosticCode, SearchDocumentDiagnosticSeverity, SearchDocumentEvidence, SearchDocumentResult, SearchDocumentsResponse, SearchDocumentWarning, } from "./search-documents.js";
+export type { SearchDocumentDiagnostic, SearchDocumentDiagnosticCode, SearchDocumentDiagnosticSeverity, SearchDocumentEvidence, SearchDocumentResult, SearchDocumentsResponse, SearchDocumentsStreamEvent, SearchDocumentWarning, } from "./search-documents.js";
 export { createSearchMinSyncDocumentsTool, SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, type SearchMinSyncDocumentsDetails, } from "./search-minsync-tool.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./system-prompt.js";

package/dist/agent/search-documents.d.ts

@@ -9,12 +9,12 @@ export type SearchDocumentDiagnosticSeverity = "info" | "warning" | "error";
  * (#21/#22) does not require another source-breaking type change; codes not yet
  * emitted are still valid members.
  */
-export type SearchDocumentDiagnosticCode = "empty-query" | "unknown-warning" | "caller-tool-dropped" | "bm25-unavailable" | "bm25-degraded-fallback" | "minsync-unavailable" | "parser-skipped" | "parser-failed" | "pdf-java-version" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "pdf-hybrid-unavailable" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited" | "unknown-datasource-skill";
+export type SearchDocumentDiagnosticCode = "empty-query" | "unknown-warning" | "caller-tool-dropped" | "minsync-unavailable" | "parser-skipped" | "parser-failed" | "pdf-java-version" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "pdf-hybrid-unavailable" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited" | "unknown-datasource-skill";
 export interface SearchDocumentDiagnostic {
     readonly code: SearchDocumentDiagnosticCode;
     readonly severity: SearchDocumentDiagnosticSeverity;
     readonly message: string;
-    /** Component label (e.g. "sanitizer", "bm25") or opaque virtual path — never a real filesystem path. */
+    /** Component label (e.g. "sanitizer", "minsync") or opaque virtual path — never a real filesystem path. */
     readonly source?: string;
 }
 export interface SearchDocumentEvidence {
@@ -43,6 +43,15 @@ export interface SearchDocumentsResponse {
      */
     readonly diagnostics?: readonly SearchDocumentDiagnostic[];
 }
+export type SearchDocumentsStreamEvent = {
+    readonly type: "progress";
+    readonly sessionId: string;
+    readonly query: string;
+    readonly text: string;
+} | {
+    readonly type: "complete";
+    readonly response: SearchDocumentsResponse;
+};
 type SearchSessions = Map<string, {
     query: string;
     registry: Map<number, CuratedResult>;

package/dist/cli/commands/evidence.d.ts

@@ -0,0 +1,3 @@
+import type { CommandContext } from "./types.js";
+/** Show the persisted source/chunk evidence attached to a search session. */
+export declare function runEvidence(ctx: CommandContext): Promise<number>;

package/dist/cli/commands/index.d.ts

@@ -1,7 +1,7 @@
 import type { CommandContext } from "./types.js";
 /**
  * Run the `autorag index` command. `reset` removes the parsed mirror
- * MinSync, and leftover legacy BM25 index directories under `.autorag`
+ * and MinSync indexes
  * (preserving `bin`, `datasources`, and the memory file). `rebuild` resets then re-runs a forced
  * refresh. Returns 0 on success, 2 for usage/decline errors, 1 for runtime
  * errors including path-escape guard violations.

package/dist/cli/commands/refresh.d.ts

@@ -2,13 +2,12 @@ import { type RefreshMethod } from "../../agent/agent.js";
 import type { CommandContext } from "./types.js";
 /**
  * `autorag refresh` — parse configured search paths and resync every active
- * index (parsed mirror, BM25, MinSync, datasources, jikji). Model-free: no LLM
+ * index (parsed mirror, MinSync, datasources, jikji). Model-free: no LLM
  * is constructed. Output is rendered through the path-opaque refresh renderer;
  * the raw result (which carries absolute `indexPath`) is never printed.
  *
- * `--method bm25,minsync,parsed` restricts which methods run. When omitted, all
- * methods run. Parsed mirrors are always synced when BM25 or MinSync is selected
- * (they index over the parsed mirrors).
+ * `--method minsync,parsed` restricts which methods run. When omitted, all
+ * methods run. Parsed mirrors are always synced when MinSync is selected.
  */
 export declare function runRefresh(ctx: CommandContext): Promise<number>;
 /**

package/dist/cli/commands/search.d.ts

@@ -21,10 +21,10 @@ export declare function classifySearchHealthHint(error: unknown): SearchHealthHi
 /**
  * Search-command dependencies. Tests inject `agentFactory` to bypass real
  * model construction and AutoRAGAgent instantiation, returning a stub whose
- * `searchDocuments` resolves a canned {@link SearchDocumentsResponse}.
+ * `searchDocumentsStream` yields progress and a canned completion.
  */
 export interface SearchDeps {
-    agentFactory?: (opts: AutoRAGAgentOptions) => Pick<AutoRAGAgent, "searchDocuments">;
+    agentFactory?: (opts: AutoRAGAgentOptions) => Pick<AutoRAGAgent, "searchDocumentsStream">;
     modelResolver?: (config: CliConfig) => ResolvedAgentModel;
 }
 /**

package/dist/cli/commands/tui.d.ts

@@ -24,7 +24,7 @@ export interface TuiPresenter {
     status(): string;
     reset(): void;
 }
-type TuiAgent = Pick<AutoRAGAgent, "searchDocuments"> & Partial<Pick<AutoRAGAgent, "subscribe" | "abort">>;
+type TuiAgent = Pick<AutoRAGAgent, "searchDocumentsStream"> & Partial<Pick<AutoRAGAgent, "subscribe" | "abort">>;
 export interface TuiDeps {
     agentFactory?: (opts?: AutoRAGAgentOptions) => TuiAgent;
     modelResolver?: (config: CliConfig) => ResolvedAgentModel;

package/dist/cli/commands/watch.d.ts

@@ -1,6 +1,6 @@
 import type { CommandContext } from "./types.js";
 /**
- * `autorag watch` — keep parsed/BM25/MinSync indexes current.
+ * `autorag watch` — keep parsed/MinSync indexes current.
  *
  * - default: long-running recursive fs watch with debounced refresh
  * - `--once`: single refresh tick (for cron / launchd / Task Scheduler)

package/dist/cli/config.d.ts

@@ -10,13 +10,6 @@ export { AUTORAG_HOME_ENV, resolveAutoRAGHome } from "../config/home.js";
 export declare class ConfigError extends Error {
     constructor(message: string);
 }
-/**
- * Typed BM25 method config persisted in `config.json`. Missing `enabled` means
- * enabled (true). `false` as a top-level value is also accepted and disables.
- */
-export interface Bm25MethodConfig {
-    enabled?: boolean;
-}
 /**
  * Typed MinSync method config persisted in `config.json`. Missing `enabled`
  * means enabled (true); `autoInstall` defaults to true. `embedder` carries
@@ -25,7 +18,6 @@ export interface Bm25MethodConfig {
 export interface MinSyncMethodConfig {
     enabled?: boolean;
     autoInstall?: boolean;
-    binaryPath?: string;
     workspacePath?: string;
     maxChunkSize?: number;
     installer?: Omit<EnsureMinSyncBinaryOptions, "root">;
@@ -33,12 +25,10 @@ export interface MinSyncMethodConfig {
 }
 /** Indexing method config as it appears in a raw config file (before normalization). */
 export interface RawIndexingMethods {
-    bm25?: Bm25MethodConfig | false;
     minSync?: MinSyncMethodConfig | false;
 }
 /** Result of {@link normalizeIndexingConfig}: always fully populated. */
 export interface NormalizedIndexingConfig {
-    bm25: Bm25MethodConfig;
     minSync: MinSyncMethodConfig;
 }
 export interface UiConfig {
@@ -82,7 +72,6 @@ export interface CliConfig {
     memoryPath: string;
     model?: AgentModelConfig;
     minSync?: MinSyncMethodConfig;
-    bm25?: Bm25MethodConfig;
     jikji?: Record<string, unknown> | false;
     parserOptions?: Record<string, unknown>;
     dupey?: {