@autorag/librarian 2.6.3 → 2.8.0

package/skills/autorag-lite-setup/SKILL.md

@@ -1,16 +1,15 @@
 ---
 name: autorag-lite-setup
-description: Initialize, index, refresh, and maintain the model-free AutoRAG Lite lifecycle (config, roots, datasources, refresh, watch, status, index reset/rebuild) without configuring any model. Use when autorag lite init/refresh/status is needed, indexes are missing or stale, or the user wants local document indexing without a search model.
+description: Install and register the model-free AutoRAG Lite MCP server, configure approved roots and datasources, build indexes, and verify MCP discovery and search. Use when autorag-mcp is missing, MCP connection fails, indexes are stale, or the user wants document search without configuring a search model.
 license: MIT
 ---
 
 # AutoRAG Lite setup
 
-Use this skill when the user wants AutoRAG's indexing lifecycle without a
-model: initialize a config, build and refresh indexes, watch roots, and inspect
-index health. Everything here is model-free. Use `autorag-setup` instead when a
-search model must be configured or repaired. Use `autorag-lite-search` for
-retrieval, reports, evidence, and feedback.
+Use this skill only to bootstrap, register, or repair AutoRAG Lite MCP:
+initialize trusted config, connect the host, build indexes, and verify search.
+Normal Lite use is MCP tool calling, not a search skill or shell command.
+Use `autorag-setup` instead when the model-backed librarian must be configured.
 
 ## Safety
 
@@ -25,16 +24,22 @@ retrieval, reports, evidence, and feedback.
   `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
   `.jikji`.
 
-## Install the CLI if needed
+## Install the package if needed
 
-The CLI is `@autorag/librarian` (`autorag`). Runtime is Node.js >= 24 or Bun.
+`@autorag/librarian` ships both `autorag` (bootstrap/maintenance) and
+`autorag-mcp` (stdio server). The executable requires Node.js >= 24 on PATH.
 
 ```bash
-command -v autorag >/dev/null || bun install -g @autorag/librarian
+command -v autorag-mcp >/dev/null || bun install -g @autorag/librarian
+command -v autorag
+command -v autorag-mcp
 autorag lite --help
 ```
 
 If Bun is unavailable, `npm install -g @autorag/librarian` is acceptable.
+If only `autorag` exists, upgrade the package rather than substituting CLI
+retrieval for MCP. For a source checkout, run `bun run build` and register the
+absolute `dist/mcp/index.js` path instead of the installed executable.
 
 ## Initialize a model-free config
 
@@ -46,7 +51,9 @@ autorag lite init \
 ```
 
 `lite init` writes the configured model-free lifecycle config. Use `--force`
-only when intentionally replacing an existing config. Explicit user paths win;
+only when intentionally replacing an existing config, and target it with an
+explicit path (`--config` or `AUTORAG_CONFIG`); `--force` refuses to replace the
+implicit `~/.autorag/config.json`. Explicit user paths win;
 otherwise propose one to three document-dense roots and get approval before
 indexing. Supported parsed formats are `md`, `markdown`, `txt`, `text`, `pdf`,
 `docx`, `pptx`, `xlsx`, `xls`, `hwp`, `hwpx`, and `eml`. Legacy `.doc` is not
@@ -57,6 +64,48 @@ Config resolution follows the usual order: `--config`, `AUTORAG_CONFIG`,
 include `AUTORAG_HOME`, `AUTORAG_CONFIG`, `AUTORAG_SEARCH_PATHS`,
 `AUTORAG_WORKSPACE`, and `AUTORAG_MEMORY_PATH`.
 
+## Register and verify the MCP connection
+
+After config and datasource setup, register the server with the host. Use
+absolute config, search root, workspace, and executable paths: hosts may start
+the server from a different working directory or with a restricted PATH.
+Resolve `command -v autorag-mcp` and substitute its absolute path below.
+Inspect an existing `autorag` registration first; keep a working registration
+and update only an outdated command or config path.
+
+```bash
+# Claude Code: project-local registration
+claude mcp add --transport stdio --scope local autorag \
+  --env AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json \
+  -- /absolute/path/to/autorag-mcp
+
+# Codex: user registration
+codex mcp add autorag \
+  --env AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json \
+  -- /absolute/path/to/autorag-mcp
+```
+
+For other hosts, use their stdio MCP configuration with the same command,
+empty arguments, and `AUTORAG_CONFIG` environment variable. The host starts
+the subprocess; do not run it as a background HTTP service. Pass any required
+credential environment-variable names through the host's secret mechanism;
+never put credential values in registration examples or logs.
+
+Reload/reconnect the host, then use MCP `tools/list` to discover the actual
+tools and schemas. Verify `autorag.status`, `autorag.refresh`, `autorag.search`,
+`autorag.search.files`, and `autorag.datasources.list` are available. Configured
+integrated datasources add their own search tools; do not assume a fixed count.
+Run `autorag.status` and `autorag.datasources.list` through MCP, then refresh
+and search a known phrase from an approved document. Registration alone is
+not proof of a connected or searchable server. After config changes, restart
+the server so datasource tools and authorization are rebuilt.
+
+`AUTORAG_MCP_READ_ONLY=1` omits refresh; build indexes with the CLI before
+connecting that mode. `AUTORAG_MCP_TOOLS` is an optional comma-separated exact
+tool allowlist; omitted tools cannot be called. Use unrestricted tools for
+initial setup unless the user intentionally requests a restricted connection.
+
+
 ## Probe and configure datasources (setup wizard)
 
 A datasource setup UI is not shipped in this build — do not recommend it for datasource
@@ -97,25 +146,26 @@ can never grant it. Store only env-var names such as `tokenEnv` or
 
 ## Build and refresh indexes
 
+After connecting, call MCP `autorag.refresh` with `{}` for an incremental run.
+Use `{"methods":["parsed","minsync"]}` to narrow indexing, or
+`{"force":true}` only when a full resync is needed. Omit `methods` to run all
+configured methods; MCP does not accept `"all"` as a method value.
+
+CLI refresh remains available for bootstrap, read-only deployments, and repair:
+
 ```bash
-autorag lite refresh --json
-autorag lite refresh --method parsed,minsync --json
-autorag lite refresh --full --json
-autorag lite refresh --force --json
+AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite refresh --json
 ```
 
-- A plain `refresh` is incremental: it syncs parsed mirrors, MinSync, Jikji,
-  and configured datasources against what changed.
-- `--full` and `--force` both request a full resync. Use them only when
-  incremental refresh is not enough, such as after config changes to roots or
-  index settings.
-- `--method <csv>` deliberately narrows the refresh. Valid values are
-  `parsed`, `minsync`, `datasources`, `jikji`, `everything` (Windows only;
-  a no-op elsewhere), `fsearch` (macOS/Linux only; a no-op elsewhere), and
-  `all`. Omit the flag to run all methods. Unknown values are rejected.
-- MinSync and Jikji auto-install on first use by default. If they are missing
-  or broken, run a full refresh or return to setup rather than silently
-  degrading to lexical-only search.
+- Refresh syncs parsed mirrors, MinSync, Jikji, and configured datasources.
+- CLI `--full` and `--force` request a full resync; MCP uses `force: true`.
+- CLI `--method <csv>` accepts `parsed`, `minsync`, `datasources`, `jikji`,
+  `everything` (Windows), `fsearch` (macOS/Linux), and `all`. MCP `methods`
+  accepts the same individual methods, not `all`. Unknown values are rejected.
+- Indexing never runs while answering: retrieval only reads what refresh built.
+  MinSync and Jikji auto-install during refresh (never during a query). If they
+  are missing or broken, run a full refresh or return to setup rather than
+  silently degrading to lexical-only search.
 - MinSync's default embedder is in-process native Qwen3 embeddings
   (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.5+): no
   embedder flags, no API key, and no external daemon (such as Ollama) are
@@ -131,18 +181,12 @@ autorag lite refresh --force --json
   and the user is told duplicate exclusion is off. Set
   `"excludeExactDuplicates": false` to index every copy.
 
-Retrieval requires a completed refresh. `autorag lite retrieve` before any
-refresh exits with code 2 and an `index-not-ready` diagnostic; a successful
-refresh is recorded even when the corpus is empty or only a non-parsed method
-was selected. Always refresh first, and refresh again when roots change.
-Once a refresh has completed, staleness is reported rather than enforced: a
-source that changed afterwards gives `"stale": true` with `stale-index`
-diagnostics (and `autorag lite status` reports `stale: true`) while retrieval
-still answers from the index. Sources the refresh deliberately skipped — exact
-duplicates, oversized or unparseable files, and AutoRAG/Jikji product artifacts
-such as `.jikji_agent_map.md` — do not count as stale. Use `autorag lite
-retrieve --refresh` to rebuild incrementally before one query, or `--strict`
-when a stale index must fail instead of answering.
+Content search requires a completed refresh. MCP `autorag.search` returns
+`isError: true` with `errorCode: "index-not-ready"` until refresh completes.
+Always refresh first, and refresh again when roots change. Stale indexes can
+still return results with `stale: true` and diagnostics; use `strict: true`
+to reject stale answers, or call `autorag.refresh` before searching. Sources
+deliberately skipped during refresh do not count as stale.
 Jikji is a discovery/indexing preparer, not a lite retrieval method.
 
 ## Watch and scheduled freshness
@@ -162,16 +206,13 @@ hourly freshness is set up.
 
 ## Status and index maintenance
 
+Use MCP `autorag.status` for health and `autorag.duplicates` for read-only
+duplicate-family inspection. No search model is required. CLI-only repair:
+
 ```bash
-autorag lite status --json
-autorag lite health --json
-autorag lite index rebuild --yes --json
-autorag lite index reset --method parsed --yes --json
-autorag lite duplicates --json
+AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite index rebuild --yes --json
+AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite index reset --method parsed --yes --json
 ```
-
-- `status` shows path-opaque corpus freshness and index health. `health` is an
-  alias of `status`; neither resolves a model.
 - `index reset` and `index rebuild` remove or rebuild only workspace
   `.autorag` indexes selected by `--method`. They never target source
   documents.
@@ -180,19 +221,34 @@ autorag lite duplicates --json
 
 ## Unavailable components and failure handling
 
-Missing optional components degrade gracefully: refresh and retrieval continue
-with diagnostics such as `minsync-unavailable` or `retrieval-method-failed`
-instead of failing the whole run. Each one quotes the underlying error verbatim,
-and `lite retrieve --json` also names the skipped surface under `unsearched`. Exit codes are 0
-on success, 2 for config or usage errors, and 1 for runtime errors. When a
-component stays unavailable after a full refresh, return to setup rather than
-accepting silently degraded search.
+Inspect MCP `isError`, `errorCode`, `diagnostics`, and `unsearched` before
+trusting results. `ok: true` with skipped surfaces is partial coverage, not
+proof that the whole corpus was searched. Preserve the underlying error text
+when reporting unavailable components such as `minsync-unavailable` or
+`retrieval-method-failed`. Repair the component rather than silently accepting
+degraded search.
+
+## Normal MCP workflow and curation lifecycle
+
+MCP results carry source, method, and content directly. After a search, the
+optional curation lifecycle is also exposed through MCP: `autorag.report`
+records a curated answer, `autorag.evidence` returns the exact chunks behind
+numbered results, and `autorag.feedback` records which numbers were useful or
+not. These accept and return JSON matching the CLI's report/evidence/feedback
+contracts; discover the exact schemas with MCP `tools/list` rather than
+restating them here. Every bracketed `[n]` citation in a report `answer` must
+be a `results[].number`; an unmatched citation is removed from the persisted
+answer and returned as a `citation-without-result` diagnostic. The CLI
+`autorag report`, `autorag evidence`, and `autorag feedback` commands remain
+available for terminal maintenance.
 
 ## Completion condition
 
-Setup is complete only when the CLI is installed, roots are approved, a
-non-secret model-free config is written, every datasource has been probed and
-the auto-configured and skipped lists reported to the user, dupey is installed
-or its absence reported, `refresh` has built the requested indexes, `status`
-reports healthy indexes, and any requested watch schedule is installed or
-verified with the user told it is active.
+Setup is complete only when the package and stdio executable are installed,
+roots are approved, trusted model-free config is written, every datasource has
+been probed and configured/skipped lists reported, and dupey is installed or
+its absence reported. The host must connect, discover tools, refresh requested
+indexes, return acceptable MCP status, and retrieve known source content via
+MCP. Any requested watch schedule must use the same config and be verified.
+Remove any previously copied Lite search skill from the host's skill directory
+so it no longer routes routine search through shell commands.

package/skills/autorag-setup/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: autorag-setup
-description: Install and configure AutoRAG, or repair its single search model, approved document roots, retrieval indexes, datasource skills, and health checks without exposing credentials. Use when autorag is missing, init/refresh/health fails, indexes are stale, or the user wants to add folders or datasources.
+description: Install and configure AutoRAG, register its Lite MCP server for external agents, or repair its single search model, approved roots, indexes, datasources, and health checks without exposing credentials. Use when AutoRAG or MCP is missing, init/refresh/health fails, indexes are stale, or the user wants to add folders or datasources.
 license: MIT
 ---
 
@@ -10,6 +10,11 @@ Use this skill when AutoRAG is unconfigured, the `autorag` CLI is missing, model
 resolution fails, indexes are missing or stale, or the user wants to change the
 document collection or datasources.
 
+For model-free external-agent search, use `autorag-lite-setup` to configure
+and register `autorag-mcp`; do not configure a search model or install a search
+skill for Lite. This skill's model setup and CLI search apply to the full
+librarian, not the Lite MCP server.
+
 ## Safety
 
 - Inspect only non-secret provider/model metadata and credential availability.
@@ -106,9 +111,20 @@ autorag init \
   --model-id MODEL
 ```
 
-If the authenticated local runtime already supplies the intended model, model
-flags may be omitted. For a custom endpoint, add `api`, `baseUrl`, and
-`apiKeyEnv` to the single `model` object in the trusted config:
+Model resolution runs through the pi runtime. A configured `provider`/`id` is
+resolved against the pi runtime catalog — the built-in providers plus any
+`~/.pi/agent/models.json`, custom, or extension providers — and credentials come
+from stored pi auth (`~/.pi/agent/auth.json`, API key or OAuth established with
+`pi /login`), then provider environment variables. When no `model` is
+configured, AutoRAG uses pi's `defaultProvider`/`defaultModel` when that
+provider has configured credentials; otherwise it falls back to the
+authenticated local codex runtime. So model flags may be omitted when either pi
+or the local runtime already supplies the intended model. Inspect what pi can
+resolve with `autorag models list` (`--available` shows only providers with
+configured credentials); it never prints credential values.
+
+For a custom endpoint, add `api`, `baseUrl`, and `apiKeyEnv` to the single
+`model` object in the trusted config:
 
 ```json
 {
@@ -123,24 +139,33 @@ flags may be omitted. For a custom endpoint, add `api`, `baseUrl`, and
 }
 ```
 
-When `provider`/`id` names a pi-ai catalog model, the catalog entry stays the
-base: `baseUrl`, `api`, and any declared `reasoning`, `input`, `contextWindow`,
-or `maxTokens` override only those fields, and the catalog's reasoning,
-thinking, and compat settings are kept. Only an id outside the catalog (private
-proxy, Ollama, LiteLLM) gets a generic text model with a 128k context window
-unless those fields are declared.
+When `provider`/`id` names a pi runtime catalog model, the catalog entry stays
+the base: `baseUrl`, `api`, and any declared `reasoning`, `input`,
+`contextWindow`, or `maxTokens` override only those fields, and the catalog's
+reasoning, thinking, and compat settings are kept. Only an id outside the
+catalog (private proxy, Ollama, LiteLLM) gets a generic text model with a 128k
+context window unless those fields are declared.
 
-Use `--force` only when intentionally replacing an existing config. Legacy cwd
+Use `--force` only when intentionally replacing an existing config, and target it
+with an explicit path (`--config` or `AUTORAG_CONFIG`); `--force` refuses to
+replace the implicit `~/.autorag/config.json`. Legacy cwd
 `autorag.config.json` is a migration source only and is never deleted by init.
 
 ### Retrieval defaults
 
 MinSync and Jikji are enabled by default. Leave them enabled unless the
-user explicitly asks otherwise. MinSync auto-installs a verified GitHub release
-into `<workspace>/.autorag/bin` on first use (`minSync.autoInstall` defaults to
-true). Set `"autoInstall": false` only when managing the binary yourself. Jikji
-auto-installs `jikji-cli` through cargo when enabled (`jikji.autoInstall`
-defaults to true; requires the Rust toolchain).
+user explicitly asks otherwise. Indexing never happens while answering: a
+question only reads indexes that `autorag refresh` (or `autorag watch`) built,
+so run a refresh after setup and whenever documents change. A never-refreshed
+workspace still answers, but without MinSync/Jikji evidence.
+
+Refresh (never a query) auto-installs the binaries: MinSync installs a verified
+GitHub release into `<workspace>/.autorag/bin` (`minSync.autoInstall` defaults
+to true), and Jikji installs `jikji-cli` through cargo (`jikji.autoInstall`
+defaults to true; requires the Rust toolchain). Set `"autoInstall": false` only
+when managing the binary yourself. Refresh is incremental: MinSync syncs only
+changed parsed mirrors, and `jikji prepare` reuses unchanged documents. Roots
+prepare in parallel.
 
 Jikji stores its prepared corpus metadata in a hidden `.jikji` directory
 inside each indexed source root — that is Jikji's native index layout and
@@ -186,6 +211,45 @@ Only store the environment-variable name, never its value. Dimension and batch
 size must be positive integers, and the dimension must match the embedder
 (default Qwen3 is 1024; legacy EmbeddingGemma is 768; text-embedding-3-small is 1536).
 
+### Jev routing and question decomposition (on by default)
+
+Leave both enabled. They are the recommended setup: they make simple questions
+fast and multi-part questions thorough.
+
+- **Jev** (`jev`, default `{ "backend": "openrouter" }`, model
+  `typesafe/jev-1.13`) runs before the fast answer. It routes each question to
+  local search, web search, or a direct answer (general knowledge or small
+  talk skips retrieval entirely), and decides whether to decompose it. After
+  the fast answer it decides whether verification is needed, so a complete,
+  evidence-backed fast answer ends the run.
+- **Question decomposition** (`queryDecomposition`, default model
+  `openrouter/qwen/qwen3.7-flash`) splits a multi-part question into at most
+  five search queries that run in parallel.
+
+Both use the user's `OPENROUTER_API_KEY`; confirm it is set (`test -n
+"$OPENROUTER_API_KEY"`, never print it) and tell the user Jev routing is on.
+Without the key, routing falls back to a single local search and the run
+always verifies, so searches still work. A `query-route-fallback` diagnostic
+(`autorag search --debug`) shows that state.
+
+```json
+{
+  "jev": { "backend": "openrouter" },
+  "queryDecomposition": { "model": { "provider": "openrouter", "id": "qwen/qwen3.7-flash" } }
+}
+```
+
+`autorag init` writes these defaults into new configs. To change them:
+
+- Jev backend: `"backend": "typesafe"` (`TYPESAFE_API_KEY`) or `"vercel"`
+  (`AI_GATEWAY_API_KEY`).
+- Decomposition model: any catalog `provider`/`id`, with the same fields as
+  the top-level `model`.
+- `"queryDecomposition": false` decomposes with the search model itself.
+- `"jev": false` turns routing off entirely. Do this only when the user
+  explicitly opts out, for example because questions must never leave the
+  machine (Jev and decomposition send the question text to OpenRouter).
+
 ### Retrieval and ingest caps
 
 `limits` bounds retrieval, baseline prefetch, and the candidate lists handed to
@@ -329,6 +393,23 @@ autorag search "summarize the collection" --top-k 3 --json --debug
   exact source chunks behind numbered results, including source, method,
   stable evidence ID, excerpt/content, chunk index, and line number.
 
+## Connect external agents through MCP
+
+When setting up AutoRAG for a coding agent, register the package's
+`autorag-mcp` stdio executable with the same absolute `AUTORAG_CONFIG` path.
+Follow `autorag-lite-setup`'s MCP registration and verification procedure:
+inspect existing host registration, use an absolute executable path,
+reload/reconnect, discover schemas with `tools/list`, and exercise
+`autorag.status`, `autorag.datasources.list`, and a known-phrase
+`autorag.search` through MCP. Restart the MCP server after config changes.
+The MCP server returns model-free source chunks; the calling agent curates
+them. It does not invoke the configured librarian model. The same server also
+exposes `autorag.report`, `autorag.evidence`, and `autorag.feedback` for the
+curation lifecycle; the matching CLI commands remain a maintenance path.
+Discover the exact schemas with MCP `tools/list`. Keep the full `autorag` skill
+only when model-backed curated search is also wanted. For MCP-only setup, use
+`autorag-lite-setup` instead of requiring live model health.
+
 ## Keep indexes fresh
 
 For continuous freshness, create or verify an OS-appropriate scheduled
@@ -358,3 +439,5 @@ installed or its absence reported, `status` is acceptable, live `health`
 passes, `refresh` builds the requested indexes, one real structured search
 succeeds, and any requested ongoing schedule is installed or verified with the
 user told it is active.
+For external-agent integration, also require successful MCP discovery and a
+known-source MCP search; CLI success alone does not prove the host connection.

package/dist/agent/bash-tool.d.ts

@@ -1,32 +0,0 @@
-import type { AgentTool } from "@earendil-works/pi-agent-core";
-import { Type } from "typebox";
-export declare const BASH_TOOL_NAME = "bash";
-declare const bashSchema: Type.TObject<{
-    command: Type.TString;
-    cwd: Type.TOptional<Type.TString>;
-    timeoutMs: Type.TOptional<Type.TInteger>;
-}>;
-export interface BashToolOptions {
-    /** Default working directory for spawned commands. */
-    readonly cwd: string;
-    /** Maximum wall-clock time per command before it is killed. */
-    readonly timeoutMs?: number;
-    /** Maximum captured output bytes before truncation. */
-    readonly maxOutputBytes?: number;
-}
-export interface BashToolDetails {
-    readonly method: "bash";
-    readonly command: string;
-    readonly aborted: boolean;
-    readonly exitCode: number | undefined;
-    readonly terminationFailed: boolean;
-    readonly timedOut: boolean;
-    readonly truncated: boolean;
-}
-/**
- * Real shell access for the librarian agent. AutoRAG navigates and reads the
- * configured collection directly through this tool (ls, find, grep, cat, …).
- * Output — including real filesystem paths — is returned to the model verbatim.
- */
-export declare function createBashTool(options: BashToolOptions): AgentTool<typeof bashSchema, BashToolDetails>;
-export {};

package/dist/cli/tui-commands.d.ts

@@ -1,16 +0,0 @@
-import type { SlashCommand } from "@earendil-works/pi-tui";
-import type { TuiSessionStore } from "./tui-session-store.js";
-export type ParsedSlashCommand = {
-    kind: "incomplete";
-} | {
-    kind: "quit";
-} | {
-    kind: "resume";
-    sessionId: string | undefined;
-} | {
-    kind: "unknown";
-    name: string;
-};
-export declare function parseSlashCommand(input: string): ParsedSlashCommand | undefined;
-export declare function createTuiSlashCommands(store: TuiSessionStore): SlashCommand[];
-export declare function renderSlashHelp(): string;

package/dist/cli/tui-session-store.d.ts

@@ -1,16 +0,0 @@
-export interface TuiSessionRecord {
-    readonly id: string;
-    readonly query: string;
-    readonly answer: string;
-    readonly trace: string;
-    readonly updatedAt: number;
-}
-export interface TuiSessionStore {
-    list(): readonly TuiSessionRecord[];
-    get(id: string): TuiSessionRecord | undefined;
-    save(session: TuiSessionRecord): void;
-}
-export declare function createFileTuiSessionStore(path: string): TuiSessionStore;
-export declare function createMergedTuiSessionStore(...stores: readonly TuiSessionStore[]): TuiSessionStore;
-export declare function renderTuiSessionList(sessions: readonly TuiSessionRecord[]): string;
-export declare function renderRestoredTuiSession(session: TuiSessionRecord): string;