@autorag/librarian 2.4.0 → 2.4.2

package/skills/autorag-lite-setup/SKILL.md

@@ -0,0 +1,182 @@
+---
+name: autorag-lite-setup
+description: Initialize, index, refresh, and maintain the model-free AutoRAG Lite lifecycle (config, roots, datasources, refresh, watch, status, index reset/rebuild) without configuring any model. Use when autorag lite init/refresh/status is needed, indexes are missing or stale, or the user wants local document indexing without a search model.
+license: MIT
+---
+
+# AutoRAG Lite setup
+
+Use this skill when the user wants AutoRAG's indexing lifecycle without a
+model: initialize a config, build and refresh indexes, watch roots, and inspect
+index health. Everything here is model-free. Use `autorag-setup` instead when a
+search model must be configured or repaired. Use `autorag-lite-search` for
+retrieval, reports, evidence, and feedback.
+
+## Safety
+
+- Inspect only non-secret config metadata: `searchPaths`, `workspacePath`,
+  `memoryPath`, `minSync`, `jikji`, `datasources`, `datasourceAccess`.
+- Never print, copy, migrate, compare, or persist credential values. Store only
+  environment-variable names such as `tokenEnv` or `apiKeyEnv`.
+- Never move, rename, edit, or delete source documents. Lite commands write
+  indexes only under the configured workspace `.autorag/` directory and
+  Jikji's per-source `.jikji/` caches.
+- Do not index system trees, app bundles, caches, credential stores,
+  `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
+  `.jikji`.
+
+## Install the CLI if needed
+
+The CLI is `@autorag/librarian` (`autorag`). Runtime is Node.js >= 24 or Bun.
+
+```bash
+command -v autorag >/dev/null || bun install -g @autorag/librarian
+autorag lite --help
+```
+
+If Bun is unavailable, `npm install -g @autorag/librarian` is acceptable.
+
+## Initialize a model-free config
+
+```bash
+autorag lite init \
+  --search-paths "/path/to/documents,/path/to/notes" \
+  --workspace "/path/to/workspace" \
+  --memory-path "/path/to/memory.json"
+```
+
+`lite init` writes the configured model-free lifecycle config. Use `--force`
+only when intentionally replacing an existing config. Explicit user paths win;
+otherwise propose one to three document-dense roots and get approval before
+indexing. Supported parsed formats are `md`, `markdown`, `txt`, `text`, `pdf`,
+`docx`, `pptx`, `xlsx`, `xls`, `hwp`, `hwpx`, and `eml`. Legacy `.doc` is not
+a supported parsed format.
+
+Config resolution follows the usual order: `--config`, `AUTORAG_CONFIG`,
+`$AUTORAG_HOME/config.json`, or `~/.autorag/config.json`. Environment overrides
+include `AUTORAG_HOME`, `AUTORAG_CONFIG`, `AUTORAG_SEARCH_PATHS`,
+`AUTORAG_WORKSPACE`, and `AUTORAG_MEMORY_PATH`.
+
+## Probe and configure datasources (setup wizard)
+
+`autorag lite ui` is still in development — do not recommend it for datasource
+setup. Configure datasources directly in trusted config, wizard-style:
+
+1. Probe every datasource for setup feasibility before asking the user
+   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`), its
+   local store or archive is present, and any credentials it needs are
+   available as environment variables or in the tool's own external
+   configuration.
+2. Auto-configure every datasource that probes feasible — write its trusted
+   `datasources` / `datasourceAccess` entries without asking. For example,
+   when Slack (`slacrawl`) and Discord (`discrawl`) are installed, set both up
+   automatically.
+3. Skip every datasource that probes infeasible (for example Notion or
+   Telegram when their CLIs are not installed) and always report the skipped
+   list to the user, with what is missing for each.
+4. Set up a skipped datasource only when the user explicitly asks for it:
+   install or authenticate the backing CLI first, then configure it.
+5. E-mail datasources (`gmail`, `mail-export`, `mailcrawl`) matter to most
+   users — always probe them and report their status, even when they end up
+   skipped.
+
+Datasource skills belong in trusted config and remain default-deny.
+
+Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
+`slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`, `gmail`,
+`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`) or connection
+aliases with `"type": "<template>"`. Unknown names are skipped with an
+`unknown-datasource-skill` warning; they do not fail config resolution.
+`datasourceAccess.allowedTags` and `allowedScopes` narrow trusted access and
+can never grant it. Store only env-var names such as `tokenEnv` or
+`apiKeyEnv`, never credential values.
+
+## Build and refresh indexes
+
+```bash
+autorag lite refresh --json
+autorag lite refresh --method parsed,minsync --json
+autorag lite refresh --full --json
+autorag lite refresh --force --json
+```
+
+- A plain `refresh` is incremental: it syncs parsed mirrors, MinSync, Jikji,
+  and configured datasources against what changed.
+- `--full` and `--force` both request a full resync. Use them only when
+  incremental refresh is not enough, such as after config changes to roots or
+  index settings.
+- `--method <csv>` deliberately narrows the refresh. Valid values are
+  `parsed`, `minsync`, `datasources`, `jikji`, and `all`. Omit the flag to run
+  all methods. Unknown values are rejected.
+- MinSync and Jikji auto-install on first use by default. If they are missing
+  or broken, run a full refresh or return to setup rather than silently
+  degrading to lexical-only search.
+- MinSync's default embedder is local EmbeddingGemma (768 dimensions, served
+  locally via Ollama): no embedder flags and no API key are needed, and no
+  corpus text leaves the machine. During setup, verify the local model is
+  available (`ollama pull embeddinggemma` with `ollama serve` running);
+  override it only when intentionally using a remote embedder.
+- Exact duplicate exclusion during refresh is enabled by default via the
+  external `dupey` CLI. Install dupey during setup when it is missing
+  (`command -v dupey || cargo install dupey`) and tell the user the feature is
+  available; when installation is impossible, refresh continues without it
+  and the user is told duplicate exclusion is off. Set
+  `"excludeExactDuplicates": false` to index every copy.
+
+Retrieval requires a completed refresh. `autorag lite retrieve` before any
+refresh exits with code 2 and an `index-not-ready` diagnostic; a successful
+refresh is recorded even when the corpus is empty or only a non-parsed method
+was selected. Always refresh first, and refresh again when roots change.
+Jikji is a discovery/indexing preparer, not a lite retrieval method.
+
+## Watch and scheduled freshness
+
+```bash
+autorag lite watch --once --json
+autorag lite watch
+```
+
+Prefer non-daemon `autorag lite watch --once` from cron, launchd, a systemd
+user timer, or Task Scheduler, hourly by default (every 1 hour; shorten only
+when the user asks for fresher indexes). Use the same config as retrieval,
+avoid overlapping runs, and keep logs outside source trees. `--immediate`
+triggers a first refresh on start and `--debounce-ms N` tunes change
+coalescing. Once the schedule is installed, tell the user right away that
+hourly freshness is set up.
+
+## Status and index maintenance
+
+```bash
+autorag lite status --json
+autorag lite health --json
+autorag lite index rebuild --yes --json
+autorag lite index reset --method parsed --yes --json
+autorag lite duplicates --json
+```
+
+- `status` shows path-opaque corpus freshness and index health. `health` is an
+  alias of `status`; neither resolves a model.
+- `index reset` and `index rebuild` remove or rebuild only workspace
+  `.autorag` indexes selected by `--method`. They never target source
+  documents.
+- `duplicates` scans duplicate document families read-only and never deletes
+  or moves files.
+
+## Unavailable components and failure handling
+
+Missing optional components degrade gracefully: refresh and retrieval continue
+with path-opaque diagnostics such as `minsync-unavailable` or
+`retrieval-method-failed` instead of failing the whole run. Exit codes are 0
+on success, 2 for config or usage errors, and 1 for runtime errors. When a
+component stays unavailable after a full refresh, return to setup rather than
+accepting silently degraded search.
+
+## Completion condition
+
+Setup is complete only when the CLI is installed, roots are approved, a
+non-secret model-free config is written, every datasource has been probed and
+the auto-configured and skipped lists reported to the user, dupey is installed
+or its absence reported, `refresh` has built the requested indexes, `status`
+reports healthy indexes, and any requested watch schedule is installed or
+verified with the user told it is active.

package/skills/autorag-setup/SKILL.md

@@ -127,7 +127,7 @@ Use `--force` only when intentionally replacing an existing config. Legacy cwd
 
 ### Retrieval defaults
 
-BM25, MinSync, and Jikji are enabled by default. Leave them enabled unless the
+MinSync and Jikji are enabled by default. Leave them enabled unless the
 user explicitly asks otherwise. MinSync auto-installs a verified GitHub release
 into `<workspace>/.autorag/bin` on first use (`minSync.autoInstall` defaults to
 true). Set `"autoInstall": false` only when managing the binary yourself. Jikji
@@ -136,11 +136,21 @@ defaults to true; requires the Rust toolchain).
 
 Exact duplicate exclusion is enabled by default. AutoRAG invokes the external
 `dupey` CLI before parsed-mirror indexing, keeps the newest filesystem copy for
-each exact canonical-text hash, and excludes older copies from the mirror. Set
-`"excludeExactDuplicates": false` to index every copy. Missing dupey is
-non-fatal; refresh continues without this optimization.
-
-Optional MinSync embedder settings:
+each exact canonical-text hash, and excludes older copies from the mirror.
+Install dupey during setup when it is missing (`command -v dupey || cargo
+install dupey`) and tell the user the feature is available; when installation
+is impossible, refresh continues without this optimization and the user is
+told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
+index every copy.
+
+MinSync's default embedder is local EmbeddingGemma (768 dimensions, served
+locally via Ollama). The default needs no embedder flags and no API key, and
+no corpus text is sent to a remote embedding service. During setup, verify the
+local model is available (`ollama pull embeddinggemma` with `ollama serve`
+running) instead of silently falling back.
+
+Override the embedder only when intentionally using a different, for example
+remote, provider:
 
 ```bash
 autorag init \
@@ -152,14 +162,32 @@ autorag init \
 ```
 
 Only store the environment-variable name, never its value. Dimension and batch
-size must be positive integers.
-
-## Configure datasource skills when requested
-
-Prefer `autorag ui --no-open` to connect datasources. It writes the same trusted
-`datasources` / `datasourceAccess` fields as a hand-edited config, stores
-env-var names rather than secrets, and prints a loopback URL (`127.0.0.1`).
-Do not bind non-loopback hosts unless the user explicitly set `ui.allowRemote`.
+size must be positive integers, and the dimension must match the embedder
+(EmbeddingGemma is 768; text-embedding-3-small is 1536).
+
+## Probe and configure datasource skills (setup wizard)
+
+`autorag ui` is still in development — do not recommend it for datasource
+setup. Configure datasources directly in trusted config, wizard-style:
+
+1. Probe every datasource for setup feasibility before asking the user
+   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`), its
+   local store or archive is present, and any credentials it needs are
+   available as environment variables or in the tool's own external
+   configuration.
+2. Auto-configure every datasource that probes feasible — write its trusted
+   `datasources` / `datasourceAccess` entries without asking. For example,
+   when Slack (`slacrawl`) and Discord (`discrawl`) are installed, set both up
+   automatically.
+3. Skip every datasource that probes infeasible (for example Notion or
+   Telegram when their CLIs are not installed) and always report the skipped
+   list to the user, with what is missing for each.
+4. Set up a skipped datasource only when the user explicitly asks for it:
+   install or authenticate the backing CLI first, then configure it.
+5. E-mail datasources (`gmail`, `mail-export`, `mailcrawl`) matter to most
+   users — always probe them and report their status, even when they end up
+   skipped.
 
 Datasource skills belong in trusted config and remain default-deny. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
@@ -175,7 +203,7 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
     "github": { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
     "google-drive": { "type": "cloud-drive", "connector": { "provider": "google-drive", "remote": "gdrive:" } },
     "archive-drive": { "type": "cloud-drive", "connector": { "remote": "archive:" } },
-    "gmail": { "connector": { "backend": "himalaya", "account": "gmail", "folder": "INBOX" } },
+    "gmail": { "connector": { "tokenEnv": "GMAIL_ACCESS_TOKEN", "labelIds": ["INBOX"] } },
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
     "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
     "rss": { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
@@ -195,6 +223,9 @@ and configured through its own Himalaya account. AutoRAG runs its local `sync`
 and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
 hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
 fails with `text array must be non-empty`.
+Use mailcrawl for Himalaya-backed IMAP/Maildir retrieval. The legacy
+`gmail` connector option `backend: "himalaya"` is no longer registered; migrate
+that configuration to an explicit `mailcrawl` datasource.
 
 ## Verify and build indexes
 
@@ -212,20 +243,25 @@ autorag search "summarize the collection" --top-k 3 --json --debug
   performs one live completion probe.
 - `health --skip-probes` is only for intentionally offline validation and does
   not prove live provider access.
-- `refresh` syncs parsed mirrors, BM25, MinSync, Jikji, and authorized
+- `refresh` syncs parsed mirrors, MinSync, Jikji, and authorized
   datasources. `--method <csv>` may deliberately narrow it
-  (`parsed,bm25,minsync,datasources,jikji,all`).
+  (`parsed,minsync,datasources,jikji,all`).
 - Use `refresh --force` for a full resync only when incremental refresh is not
   enough. Keep destructive reset/rebuild operations scoped to workspace
   `.autorag` indexes, never source documents.
+- After a search, use `autorag evidence <sessionId> --json` to inspect the
+  exact source chunks behind numbered results, including source, method,
+  stable evidence ID, excerpt/content, chunk index, and line number.
 
 ## Keep indexes fresh
 
 For continuous freshness, create or verify an OS-appropriate scheduled
-`autorag watch --once` job, typically every 15–30 minutes. Prefer cron or
-launchd on macOS, cron or a user systemd timer on Linux, and Task Scheduler on
-Windows. Use the same config as search, avoid overlapping runs, and keep logs
-outside source trees.
+`autorag watch --once` job, hourly by default (every 1 hour; shorten only when
+the user asks for fresher indexes). Prefer cron or launchd on macOS, cron or a
+user systemd timer on Linux, and Task Scheduler on Windows. Use the same
+config as search, avoid overlapping runs, and keep logs outside source trees.
+Once the schedule is installed, tell the user right away that hourly freshness
+is set up.
 
 ## Environment overrides
 
@@ -240,6 +276,9 @@ outside source trees.
 ## Completion condition
 
 Setup is complete only when the CLI is installed, roots are approved, a
-non-secret single-model config is written, `status` is acceptable, live `health`
+non-secret single-model config is written, every datasource has been probed
+and the auto-configured and skipped lists reported to the user, dupey is
+installed or its absence reported, `status` is acceptable, live `health`
 passes, `refresh` builds the requested indexes, one real structured search
-succeeds, and any requested ongoing schedule is installed or verified.
+succeeds, and any requested ongoing schedule is installed or verified with the
+user told it is active.

package/dist/agent/search-bm25-tool.d.ts

@@ -1,26 +0,0 @@
-import type { AgentTool } from "@earendil-works/pi-agent-core";
-import { Type } from "typebox";
-import type { BM25Status } from "../retrieval/methods/bm25.js";
-import type { RetrievalResult } from "../retrieval/types.js";
-export declare const SEARCH_BM25_DOCUMENTS_TOOL_NAME = "lexical_search_local_docs";
-declare const searchBM25Schema: Type.TObject<{
-    query: Type.TString;
-    topK: Type.TOptional<Type.TInteger>;
-    scope: Type.TOptional<Type.TString>;
-}>;
-export interface SearchBM25DocumentsDetails {
-    readonly method: "bm25";
-    readonly resultCount: number;
-    readonly sources: readonly string[];
-    readonly readiness: BM25Status["readiness"];
-    readonly engine: BM25Status["engine"];
-}
-export interface BM25SearchMethod {
-    getStatus(): BM25Status;
-    retrieve(query: string, options: {
-        topK?: number;
-        scope?: string;
-    }): Promise<RetrievalResult[]>;
-}
-export declare function createSearchBM25DocumentsTool(getMethod: () => BM25SearchMethod | undefined, resolveScope?: (scope: string | undefined) => string | undefined): AgentTool<typeof searchBM25Schema, SearchBM25DocumentsDetails>;
-export {};

package/dist/datasource/skills/gmail/himalaya-connector.d.ts

@@ -1,51 +0,0 @@
-/**
- * Himalaya-backed mail connector (issue #1304, IMAP path).
- *
- * Uses the external `himalaya` CLI (https://pimalaya.org) as the trusted
- * bridge to any IMAP/Maildir account it has configured — Gmail included —
- * so no OAuth token plumbing is needed here. Mirrors the katok pattern:
- * AutoRAG never opens the mailbox itself; it spawns the CLI, parses JSON
- * envelopes, and reads message bodies. Never throws; failures map onto the
- * coarse connector failure union with path/PII-opaque messages.
- */
-import { type ConnectorFetchResult, type DatasourceConnector } from "../../connector.js";
-export interface HimalayaRunResult {
-    readonly ok: boolean;
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number | null;
-}
-export type HimalayaRunner = (args: readonly string[], timeoutMs: number) => Promise<HimalayaRunResult>;
-export interface HimalayaConnectorOptions {
-    /** Path to the himalaya binary. Default bare `himalaya` PATH lookup. */
-    readonly binaryPath?: string;
-    /** Himalaya account name (`himalaya account list`). Default account when omitted. */
-    readonly account?: string;
-    /** Mail folder, e.g. `INBOX`. Himalaya's default folder when omitted. */
-    readonly folder?: string;
-    /** Envelopes per listing page. Default 100. */
-    readonly pageSize?: number;
-    /** Max messages whose bodies are fetched per run. Default 50. */
-    readonly maxDocuments?: number;
-    /**
-     * Workspace root for the connector's envelope fingerprint state. When set,
-     * subsequent refreshes fetch only new or changed envelopes and update the
-     * shared chunk store incrementally.
-     */
-    readonly workspaceRoot?: string;
-    /** Per-spawn timeout. Default 30s (IMAP fetches can be slow). */
-    readonly timeoutMs?: number;
-    /** Injectable process runner for tests. */
-    readonly runner?: HimalayaRunner;
-    /** Explicit operator-owned config path, passed read-only to Himalaya. */
-    readonly configPath?: string;
-}
-export declare class HimalayaConnector implements DatasourceConnector {
-    private readonly options;
-    private readonly runner;
-    constructor(options?: HimalayaConnectorOptions);
-    fetch(): Promise<ConnectorFetchResult>;
-    private loadState;
-    private saveState;
-    private statePath;
-}

package/dist/retrieval/methods/bm25.d.ts

@@ -1,23 +0,0 @@
-export type BM25ReadinessState = "disabled" | "dependency_unavailable" | "index_missing" | "indexing" | "ready" | "error";
-/** Local BM25 is always MinSync-backed when enabled. */
-export type BM25Engine = "minsync" | "none";
-export interface BM25SyncResult {
-    readonly indexPath: string;
-    readonly indexedChunks: number;
-    readonly readiness: BM25ReadinessState;
-    readonly engine: BM25Engine;
-}
-export interface BM25Status {
-    readonly readiness: BM25ReadinessState;
-    readonly engine: BM25Engine;
-    readonly message?: string;
-}
-/** Legacy AutoRAG-local BM25 artifact directory. Never an active index. */
-export declare const BM25_SUBDIR: string;
-export declare class BM25UnavailableError extends Error {
-    readonly readiness: BM25ReadinessState;
-    constructor(readiness: BM25ReadinessState, message: string);
-}
-/** Remove leftover `.autorag/bm25` artifacts from the pre-MinSync BM25 path. */
-export declare function removeLegacyBm25Artifacts(root: string): void;
-export declare function hasLegacyBm25Artifacts(root: string): boolean;