@autorag/librarian 2.4.1 → 2.4.2

package/dist/minsync/client.d.ts

@@ -5,6 +5,7 @@ export interface MinSyncClientOptions {
     readonly embedder?: MinSyncEmbedderConfig;
     readonly maxChunkSize?: number;
 }
+/** MinSync v0.4.2 supports vector, BM25, and hybrid query modes. */
 export type MinSyncQueryMode = "vector" | "bm25" | "hybrid";
 export declare class MinSyncClient {
     private readonly binaryPath;
@@ -12,7 +13,7 @@ export declare class MinSyncClient {
     private readonly embedder;
     private readonly maxChunkSize;
     constructor(options: MinSyncClientOptions);
-    sync(): Promise<MinSyncSyncResult>;
+    sync(force?: boolean): Promise<MinSyncSyncResult>;
     query(text: string, topK: number, mode?: MinSyncQueryMode): Promise<readonly MinSyncQueryHit[]>;
     private spawn;
 }

package/dist/minsync/installer.d.ts

@@ -1,4 +1,5 @@
-export declare const MINSYNC_VERSION = "0.3.0";
+export declare const MINSYNC_VERSION = "0.4.2";
+export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
 export interface MinSyncReleaseAsset {
     readonly name: string;
     readonly downloadUrl: string;
@@ -18,6 +19,11 @@ export interface EnsureMinSyncBinaryOptions {
     readonly arch?: NodeJS.Architecture;
     readonly releaseProvider?: () => Promise<MinSyncRelease>;
     readonly assetInstaller?: (asset: MinSyncReleaseAsset, destination: string) => Promise<void>;
+    /** Test seam: cargo-first installer. Return undefined to fall back to GitHub. */
+    readonly cargoInstaller?: (destination: string) => Promise<InstalledMinSyncBinary | undefined>;
+    /** Test seam: locate cargo on PATH. */
+    readonly cargoLocator?: (env: NodeJS.ProcessEnv) => string | undefined;
+    readonly env?: NodeJS.ProcessEnv;
 }
 export declare function ensureMinSyncBinary(options: EnsureMinSyncBinaryOptions): Promise<InstalledMinSyncBinary>;
 export declare function fetchLatestMinSyncRelease(): Promise<MinSyncRelease>;

package/dist/minsync/method.d.ts

@@ -23,11 +23,14 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
     private readonly embedder;
     private readonly maxChunkSize;
     private readonly mode;
+    private installFailed;
     constructor(options: MinSyncVectorMethodOptions);
     describe(): RetrievalMethodDescriptor;
-    sync(): Promise<MinSyncSyncResult>;
-    /** MinSync is resolved from PATH, then the workspace cache. */
+    sync(force?: boolean): Promise<MinSyncSyncResult>;
+    /** Report unavailable binaries without treating auto-install as already failed. */
     isBinaryMissing(): boolean;
+    /** Status distinguishes an explicit missing override from deferred resolution. */
+    isExplicitBinaryMissing(): boolean;
     /** A vector index is ready only after MinSync has written its cursor. */
     isReady(): boolean;
     retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;

package/dist/mirror/index.d.ts

@@ -1,5 +1,5 @@
 export type { ParsedMirrorEntry, ParsedMirrorIndex } from "./index-store.js";
 export { emptyMirrorIndex, loadMirrorIndex, saveMirrorIndex } from "./index-store.js";
-export { parsedMirrorIndexPath, parsedMirrorRoot, parsedOutputPath } from "./paths.js";
+export { parsedMirrorIndexPath, parsedMirrorRoot, parsedOutputPath, refreshReadinessPath } from "./paths.js";
 export type { ParsedMirrorDiagnostic, ParsedMirrorDiagnosticCode, ParsedMirrorSyncOptions, ParsedMirrorSyncResult, } from "./sync.js";
 export { DEFAULT_MAX_SOURCE_BYTES, detectMirrorStaleness, syncParsedMirrors } from "./sync.js";

package/dist/mirror/paths.d.ts

@@ -1,6 +1,8 @@
 export declare const PARSED_MIRROR_SUBDIR: string;
 export declare const PARSED_FILES_SUBDIR = "files";
 export declare const PARSED_INDEX_FILE = "index.json";
+export declare const REFRESH_READINESS_FILE = "refresh-complete.json";
 export declare function parsedMirrorRoot(root: string): string;
 export declare function parsedMirrorIndexPath(root: string): string;
 export declare function parsedOutputPath(root: string, virtualPath: string): string;
+export declare function refreshReadinessPath(root: string): string;

package/dist/retrieval/engine.d.ts

@@ -0,0 +1,123 @@
+/**
+ * RetrievalEngine — standalone typed seam for model-free retrieval.
+ *
+ * Wraps the existing retrieval pipeline (registry, parallel retriever,
+ * datasource result filter, and merger) into a single public entry point.
+ * Exposes a deterministic, self-contained retrieval API with diagnostics.
+ *
+ * Security invariants (delegated to {@link DatasourceAccessContext} and
+ * {@link DatasourceResultFilter}):
+ *  - Access is default-deny: when no trusted allow-tags are configured,
+ *    every datasource method result is dropped.
+ *  - Deny is always an explicit empty result array, never `undefined`.
+ *  - Model/tool arguments (`allowedTags`, `allowedScopes`) cannot widen the
+ *    trusted access context constructed during engine creation.
+ */
+import { DatasourceAccessContext, type DatasourceAccessContextOptions } from "../datasource/access-context.js";
+import { RetrievalMethodRegistry } from "./registry.js";
+import type { RetrievalDiagnostic, RetrievalMethod, RetrievalOptions, RetrievalResult } from "./types.js";
+/** Options for constructing a standalone {@link RetrievalEngine}. */
+export interface RetrievalEngineOptions {
+    /**
+     * Datasource access configuration. Default-deny when omitted.
+     * @default { allowedTags: undefined, allowedScopes: undefined }
+     */
+    readonly datasourceAccess?: DatasourceAccessContextOptions;
+    /** Default `topK` when the caller omits it. @default 20 */
+    readonly defaultTopK?: number;
+    /** Default deduplication flag. @default true */
+    readonly defaultDedup?: boolean;
+    /**
+     * Optional predicate that returns `true` when the MinSync binary is missing.
+     * When set, the engine checks every method named `"minsync"`: if the binary
+     * is unavailable the method returns empty results without throwing, the engine
+     * emits a `minsync-unavailable` diagnostic with full method-agnostic path
+     * preservation — matching the existing {@link AutoRAGAgent} behavior.
+     */
+    readonly isMinSyncBinaryMissing?: () => boolean;
+}
+/**
+ * Model-free retrieval pipeline.
+ *
+ * The engine composes the standard AutoRAG retrieval chain:
+ * ```
+ * methods = registry.list()
+ * byMethod = retriever.retrieveWithDiagnostics(methods, query, options)
+ * filtered = filter.filter(byMethod, methods, ctx, options.scope)
+ * merged = merger.merge(filtered, { topK, dedup: true })
+ * → { results: merged, diagnostics }
+ * ```
+ *
+ * Exposes two paths:
+ *  1. `retrieve(query, options?)` — merged results + diagnostics (the standard
+ *     pipeline).
+ *  2. `retrieveByMethod(query, options?)` — per-method results (unmerged) with
+ *     diagnostics, for callers that need the raw per-method view.
+ *
+ * The engine does NOT handle refresh lifecycle, MinSync sync, Jikji, or
+ * agent-level concerns. It is purely the retrieval composition seam.
+ */
+export declare class RetrievalEngine {
+    private readonly registry;
+    private readonly retriever;
+    private readonly filter;
+    private readonly merger;
+    private readonly accessContext;
+    private readonly defaultTopK;
+    private readonly defaultDedup;
+    private readonly isMinSyncBinaryMissing;
+    constructor(options?: RetrievalEngineOptions);
+    /** The underlying method registry. Intentionally public for tooling. */
+    getMethodRegistry(): RetrievalMethodRegistry;
+    /** The underlying datasource access context. Read-only for inspection. */
+    getAccessContext(): DatasourceAccessContext;
+    /**
+     * Register a retrieval method. Overrides the registered-once check from
+     * {@link RetrievalMethodRegistry} per caller requirement.
+     *
+     * @throws {Error} if a method with the same name is already registered.
+     */
+    register(method: RetrievalMethod): void;
+    /**
+     * Register multiple methods in one call. Stop-on-first-error semantics:
+     * registration fails atomically if any method conflicts with an already-
+     * registered name; earlier methods in the batch are not retained.
+     *
+     * @throws {Error} if any method name duplicates an existing or in-batch registration.
+     */
+    registerMany(methods: readonly RetrievalMethod[]): void;
+    /**
+     * Run the full retrieval pipeline — parallel retrieve, datasource filter,
+     * merge — and return merged results with diagnostics.
+     */
+    retrieve(query: string, options?: RetrievalOptions): Promise<{
+        results: RetrievalResult[];
+        diagnostics: RetrievalDiagnostic[];
+    }>;
+    /**
+     * Run retrieval and return results keyed per method (unmerged) with
+     * diagnostics. Unlike {@link retrieve}, this does NOT merge or sort —
+     * callers get the raw per-method view after datasource filtering.
+     */
+    retrieveByMethod(query: string, options?: RetrievalOptions): Promise<{
+        byMethod: Map<string, RetrievalResult[]>;
+        diagnostics: RetrievalDiagnostic[];
+    }>;
+    /**
+     * Post-check for MinSync binary-missing diagnostic.
+     *
+     * When a registered method named `"minsync"` fails to throw (the method returns
+     * `[]` because the binary is missing), {@link ParallelRetriever} cannot record
+     * a diagnostic. This method checks whether the minsync binary is unavailable
+     * and emits a `minsync-unavailable` diagnostic when every minsync method's
+     * result set is empty, exactly matching the existing {@link AutoRAGAgent}
+     * behavior (see `agent.src/agent/agent.ts` `retrieveWithDiagnostics`).
+     */
+    private appendMinSyncUnavailableDiagnostic;
+    /**
+     * Build a datasource access context scoped to the caller's options.
+     * Model/tool arguments (allowedTags, allowedScopes) from options can only
+     * further restrict the trusted context, never widen it.
+     */
+    private accessContextFor;
+}

package/dist/retrieval/index.d.ts

@@ -1,3 +1,4 @@
+export { RetrievalEngine, type RetrievalEngineOptions } from "./engine.js";
 export type { MergeOptions } from "./merger.js";
 export { ParallelRetriever, ResultMerger } from "./merger.js";
 export { RetrievalMethodRegistry } from "./registry.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.4.1",
+  "version": "2.4.2",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -14,6 +14,10 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./core": {
+      "types": "./dist/core.d.ts",
+      "default": "./dist/core.js"
     }
   },
   "bin": {
@@ -28,7 +32,7 @@
     "access": "public"
   },
   "scripts": {
-    "build": "bun build src/index.ts src/cli/index.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
+    "build": "bun build src/index.ts src/core.ts src/cli/index.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
     "prepublishOnly": "bun run typecheck && bun run build",
     "prepare": "bun run build",
     "test": "node scripts/run-test-files.mjs",

package/skills/autorag-lite-search/SKILL.md

@@ -0,0 +1,155 @@
+---
+name: autorag-lite-search
+description: Retrieve documents from an indexed AutoRAG Lite corpus without a model, then persist externally curated reports, inspect evidence, and record numbered feedback. Use when the user wants model-free search, scoped or tag-narrowed retrieval, or the report/evidence/feedback workflow over local documents and authorized datasources.
+license: MIT
+---
+
+# AutoRAG Lite search
+
+Use this skill when AutoRAG Lite is already configured and indexed, and the
+user wants model-free retrieval, or wants to persist an externally curated
+report, inspect its evidence, or record feedback. Use `autorag-lite-setup`
+when config, indexing, or refresh is missing or broken. Use the `autorag`
+skill when a model-backed curated answer is requested.
+
+AutoRAG Lite never mutates sources. Retrieval reads local source documents and
+authorized datasource content; writes go only to workspace `.autorag/` indexes,
+the memory file, and Jikji `.jikji/` caches.
+
+## Preflight
+
+Retrieval requires a completed refresh. If `autorag lite retrieve` returns
+exit code 2 with an `index-not-ready` diagnostic, run `autorag lite refresh`
+(or `autorag refresh`) first, then retry. Confirm freshness with:
+
+```bash
+autorag lite status --json
+```
+
+`status` is model-free and path-opaque.
+
+## Retrieve
+
+```bash
+autorag lite retrieve "key findings in the Q3 report" --top-k 5 --json
+```
+
+The JSON envelope is:
+
+```json
+{
+  "ok": true,
+  "query": "key findings in the Q3 report",
+  "results": [
+    {
+      "number": 1,
+      "source": "<opaque source identifier>",
+      "method": "minsync",
+      "score": 0.83,
+      "metadata": {},
+      "content": "..."
+    }
+  ],
+  "diagnostics": []
+}
+```
+
+- `--top-k N` must be a positive integer. Invalid values exit with code 2 and
+  an `{"ok": false, "error": "..."}` rejection envelope.
+- Before any refresh, the envelope is `{"ok": false, "query": ..., "diagnostics":
+  [{"code": "index-not-ready", "severity": "error", ...}]}` with exit code 2.
+- Each result carries a provenance pair: the `source` identity and the
+  `method` that produced it. Source identities are source-native: real file
+  paths for local files, datasource identities for datasource results. Never
+  rewrite, guess, or flatten them.
+- `diagnostics` are path-opaque: `source` is a component or method label,
+  never a real filesystem path. Codes include `retrieval-method-failed` and
+  `minsync-unavailable`. A degraded component produces a diagnostic, not a
+  failed run; check `diagnostics` before trusting an empty result set.
+- `--debug` adds diagnostic detail to human output without printing real
+  filesystem paths.
+- Exit codes: 0 on success (results may be empty), 2 for usage, config, or
+  not-ready errors, 1 for runtime errors.
+
+Retrieval methods run in parallel and merge: parsed mirrors and MinSync
+lexical/vector/hybrid methods, plus configured datasource methods. Jikji is a
+local discovery/indexing preparer, not a retrieval method in the lite registry.
+`--method` selection applies to refresh, not retrieve; retrieval always queries
+every active retrieval method.
+
+## Scope and tag narrowing (default-deny)
+
+Datasource access is default-deny from trusted config
+(`datasourceAccess.allowedTags`, `allowedScopes`). Retrieval flags can only
+narrow that access, never widen it:
+
+- `--scope SCOPE` narrows datasource retrieval to a requested sub-path; it
+  cannot grant access to anything the config does not already allow.
+- `--tags tag1,tag2` further narrows already-authorized datasource results by
+  intersecting with configured allowed tags; it never grants new access.
+
+## Persisting a curated report
+
+An external agent curates the raw retrieval results, then persists the
+structured report so evidence and feedback commands work against it:
+
+```bash
+autorag lite report "key findings in the Q3 report" --input report.json --json
+cat report.json | autorag lite report "key findings in the Q3 report" --json
+```
+
+The input is the `emit_autorag_results` JSON schema: `answer`, numbered
+`results` (with `number`, `title`, `summary`, `confidence` in `[0, 1]`,
+optional `source`), and a `mapping` whose entries map each result number to
+its `method` and `source`, plus optional `evidenceRefs` and `warnings`.
+Result and mapping numbers must be one-to-one positive integers. Evidence
+confidence values must be in `[0, 1]`. Invalid input exits with code 2 and
+`{"ok": false, "error": "..."}`.
+
+`source` values in a report are opaque identifiers carried from retrieval
+output. Report persistence performs no filesystem reads against them; it
+stores the identifiers as given. Preserve the exact source-native identities
+from the retrieve envelope.
+
+On success the JSON envelope is:
+
+```json
+{
+  "ok": true,
+  "sessionId": "<uuid>",
+  "query": "key findings in the Q3 report",
+  "answer": "...",
+  "resultCount": 3
+}
+```
+
+Use the returned `sessionId` for evidence inspection and feedback.
+
+## Evidence and feedback
+
+```bash
+autorag evidence <sessionId> --result 1 --json
+autorag feedback <sessionId> --useful 1,3 --not-useful 2 --json
+```
+
+- `evidence` shows the persisted source, retrieval method, stable evidence ID,
+  excerpt or content, and any `chunkIndex`, `lineNumber`, and
+  `retrievalResultId` behind a numbered result. Omit `--result` to inspect the
+  whole session.
+- `feedback` records numbered usefulness so retrieval memory can learn.
+  Numbers refer to the report's result numbers. Supply at least one feedback
+  list.
+- Preserve real source mapping and numbered identifiers end to end: retrieve
+  output feeds the report, the report's session feeds evidence and feedback.
+
+## Rules
+
+- Refresh before retrieval; refresh again when roots change.
+- Use only configured and approved search paths and datasources.
+- Never move, rename, edit, or delete source documents.
+- Never expose provider credentials or authentication payloads.
+- Treat report `source` values as opaque: persist them verbatim, never read
+  the filesystem through them.
+- Keep diagnostics path-opaque; do not reconstruct or disclose real paths
+  from them.
+- Prefer `--json` whenever another agent consumes the output.

package/skills/autorag-lite-setup/SKILL.md

@@ -0,0 +1,182 @@
+---
+name: autorag-lite-setup
+description: Initialize, index, refresh, and maintain the model-free AutoRAG Lite lifecycle (config, roots, datasources, refresh, watch, status, index reset/rebuild) without configuring any model. Use when autorag lite init/refresh/status is needed, indexes are missing or stale, or the user wants local document indexing without a search model.
+license: MIT
+---
+
+# AutoRAG Lite setup
+
+Use this skill when the user wants AutoRAG's indexing lifecycle without a
+model: initialize a config, build and refresh indexes, watch roots, and inspect
+index health. Everything here is model-free. Use `autorag-setup` instead when a
+search model must be configured or repaired. Use `autorag-lite-search` for
+retrieval, reports, evidence, and feedback.
+
+## Safety
+
+- Inspect only non-secret config metadata: `searchPaths`, `workspacePath`,
+  `memoryPath`, `minSync`, `jikji`, `datasources`, `datasourceAccess`.
+- Never print, copy, migrate, compare, or persist credential values. Store only
+  environment-variable names such as `tokenEnv` or `apiKeyEnv`.
+- Never move, rename, edit, or delete source documents. Lite commands write
+  indexes only under the configured workspace `.autorag/` directory and
+  Jikji's per-source `.jikji/` caches.
+- Do not index system trees, app bundles, caches, credential stores,
+  `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
+  `.jikji`.
+
+## Install the CLI if needed
+
+The CLI is `@autorag/librarian` (`autorag`). Runtime is Node.js >= 24 or Bun.
+
+```bash
+command -v autorag >/dev/null || bun install -g @autorag/librarian
+autorag lite --help
+```
+
+If Bun is unavailable, `npm install -g @autorag/librarian` is acceptable.
+
+## Initialize a model-free config
+
+```bash
+autorag lite init \
+  --search-paths "/path/to/documents,/path/to/notes" \
+  --workspace "/path/to/workspace" \
+  --memory-path "/path/to/memory.json"
+```
+
+`lite init` writes the configured model-free lifecycle config. Use `--force`
+only when intentionally replacing an existing config. Explicit user paths win;
+otherwise propose one to three document-dense roots and get approval before
+indexing. Supported parsed formats are `md`, `markdown`, `txt`, `text`, `pdf`,
+`docx`, `pptx`, `xlsx`, `xls`, `hwp`, `hwpx`, and `eml`. Legacy `.doc` is not
+a supported parsed format.
+
+Config resolution follows the usual order: `--config`, `AUTORAG_CONFIG`,
+`$AUTORAG_HOME/config.json`, or `~/.autorag/config.json`. Environment overrides
+include `AUTORAG_HOME`, `AUTORAG_CONFIG`, `AUTORAG_SEARCH_PATHS`,
+`AUTORAG_WORKSPACE`, and `AUTORAG_MEMORY_PATH`.
+
+## Probe and configure datasources (setup wizard)
+
+`autorag lite ui` is still in development — do not recommend it for datasource
+setup. Configure datasources directly in trusted config, wizard-style:
+
+1. Probe every datasource for setup feasibility before asking the user
+   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`), its
+   local store or archive is present, and any credentials it needs are
+   available as environment variables or in the tool's own external
+   configuration.
+2. Auto-configure every datasource that probes feasible — write its trusted
+   `datasources` / `datasourceAccess` entries without asking. For example,
+   when Slack (`slacrawl`) and Discord (`discrawl`) are installed, set both up
+   automatically.
+3. Skip every datasource that probes infeasible (for example Notion or
+   Telegram when their CLIs are not installed) and always report the skipped
+   list to the user, with what is missing for each.
+4. Set up a skipped datasource only when the user explicitly asks for it:
+   install or authenticate the backing CLI first, then configure it.
+5. E-mail datasources (`gmail`, `mail-export`, `mailcrawl`) matter to most
+   users — always probe them and report their status, even when they end up
+   skipped.
+
+Datasource skills belong in trusted config and remain default-deny.
+
+Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
+`slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`, `gmail`,
+`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`) or connection
+aliases with `"type": "<template>"`. Unknown names are skipped with an
+`unknown-datasource-skill` warning; they do not fail config resolution.
+`datasourceAccess.allowedTags` and `allowedScopes` narrow trusted access and
+can never grant it. Store only env-var names such as `tokenEnv` or
+`apiKeyEnv`, never credential values.
+
+## Build and refresh indexes
+
+```bash
+autorag lite refresh --json
+autorag lite refresh --method parsed,minsync --json
+autorag lite refresh --full --json
+autorag lite refresh --force --json
+```
+
+- A plain `refresh` is incremental: it syncs parsed mirrors, MinSync, Jikji,
+  and configured datasources against what changed.
+- `--full` and `--force` both request a full resync. Use them only when
+  incremental refresh is not enough, such as after config changes to roots or
+  index settings.
+- `--method <csv>` deliberately narrows the refresh. Valid values are
+  `parsed`, `minsync`, `datasources`, `jikji`, and `all`. Omit the flag to run
+  all methods. Unknown values are rejected.
+- MinSync and Jikji auto-install on first use by default. If they are missing
+  or broken, run a full refresh or return to setup rather than silently
+  degrading to lexical-only search.
+- MinSync's default embedder is local EmbeddingGemma (768 dimensions, served
+  locally via Ollama): no embedder flags and no API key are needed, and no
+  corpus text leaves the machine. During setup, verify the local model is
+  available (`ollama pull embeddinggemma` with `ollama serve` running);
+  override it only when intentionally using a remote embedder.
+- Exact duplicate exclusion during refresh is enabled by default via the
+  external `dupey` CLI. Install dupey during setup when it is missing
+  (`command -v dupey || cargo install dupey`) and tell the user the feature is
+  available; when installation is impossible, refresh continues without it
+  and the user is told duplicate exclusion is off. Set
+  `"excludeExactDuplicates": false` to index every copy.
+
+Retrieval requires a completed refresh. `autorag lite retrieve` before any
+refresh exits with code 2 and an `index-not-ready` diagnostic; a successful
+refresh is recorded even when the corpus is empty or only a non-parsed method
+was selected. Always refresh first, and refresh again when roots change.
+Jikji is a discovery/indexing preparer, not a lite retrieval method.
+
+## Watch and scheduled freshness
+
+```bash
+autorag lite watch --once --json
+autorag lite watch
+```
+
+Prefer non-daemon `autorag lite watch --once` from cron, launchd, a systemd
+user timer, or Task Scheduler, hourly by default (every 1 hour; shorten only
+when the user asks for fresher indexes). Use the same config as retrieval,
+avoid overlapping runs, and keep logs outside source trees. `--immediate`
+triggers a first refresh on start and `--debounce-ms N` tunes change
+coalescing. Once the schedule is installed, tell the user right away that
+hourly freshness is set up.
+
+## Status and index maintenance
+
+```bash
+autorag lite status --json
+autorag lite health --json
+autorag lite index rebuild --yes --json
+autorag lite index reset --method parsed --yes --json
+autorag lite duplicates --json
+```
+
+- `status` shows path-opaque corpus freshness and index health. `health` is an
+  alias of `status`; neither resolves a model.
+- `index reset` and `index rebuild` remove or rebuild only workspace
+  `.autorag` indexes selected by `--method`. They never target source
+  documents.
+- `duplicates` scans duplicate document families read-only and never deletes
+  or moves files.
+
+## Unavailable components and failure handling
+
+Missing optional components degrade gracefully: refresh and retrieval continue
+with path-opaque diagnostics such as `minsync-unavailable` or
+`retrieval-method-failed` instead of failing the whole run. Exit codes are 0
+on success, 2 for config or usage errors, and 1 for runtime errors. When a
+component stays unavailable after a full refresh, return to setup rather than
+accepting silently degraded search.
+
+## Completion condition
+
+Setup is complete only when the CLI is installed, roots are approved, a
+non-secret model-free config is written, every datasource has been probed and
+the auto-configured and skipped lists reported to the user, dupey is installed
+or its absence reported, `refresh` has built the requested indexes, `status`
+reports healthy indexes, and any requested watch schedule is installed or
+verified with the user told it is active.

package/skills/autorag-setup/SKILL.md

@@ -136,11 +136,21 @@ defaults to true; requires the Rust toolchain).
 
 Exact duplicate exclusion is enabled by default. AutoRAG invokes the external
 `dupey` CLI before parsed-mirror indexing, keeps the newest filesystem copy for
-each exact canonical-text hash, and excludes older copies from the mirror. Set
-`"excludeExactDuplicates": false` to index every copy. Missing dupey is
-non-fatal; refresh continues without this optimization.
-
-Optional MinSync embedder settings:
+each exact canonical-text hash, and excludes older copies from the mirror.
+Install dupey during setup when it is missing (`command -v dupey || cargo
+install dupey`) and tell the user the feature is available; when installation
+is impossible, refresh continues without this optimization and the user is
+told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
+index every copy.
+
+MinSync's default embedder is local EmbeddingGemma (768 dimensions, served
+locally via Ollama). The default needs no embedder flags and no API key, and
+no corpus text is sent to a remote embedding service. During setup, verify the
+local model is available (`ollama pull embeddinggemma` with `ollama serve`
+running) instead of silently falling back.
+
+Override the embedder only when intentionally using a different, for example
+remote, provider:
 
 ```bash
 autorag init \
@@ -152,14 +162,32 @@ autorag init \
 ```
 
 Only store the environment-variable name, never its value. Dimension and batch
-size must be positive integers.
-
-## Configure datasource skills when requested
-
-Prefer `autorag ui --no-open` to connect datasources. It writes the same trusted
-`datasources` / `datasourceAccess` fields as a hand-edited config, stores
-env-var names rather than secrets, and prints a loopback URL (`127.0.0.1`).
-Do not bind non-loopback hosts unless the user explicitly set `ui.allowRemote`.
+size must be positive integers, and the dimension must match the embedder
+(EmbeddingGemma is 768; text-embedding-3-small is 1536).
+
+## Probe and configure datasource skills (setup wizard)
+
+`autorag ui` is still in development — do not recommend it for datasource
+setup. Configure datasources directly in trusted config, wizard-style:
+
+1. Probe every datasource for setup feasibility before asking the user
+   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`), its
+   local store or archive is present, and any credentials it needs are
+   available as environment variables or in the tool's own external
+   configuration.
+2. Auto-configure every datasource that probes feasible — write its trusted
+   `datasources` / `datasourceAccess` entries without asking. For example,
+   when Slack (`slacrawl`) and Discord (`discrawl`) are installed, set both up
+   automatically.
+3. Skip every datasource that probes infeasible (for example Notion or
+   Telegram when their CLIs are not installed) and always report the skipped
+   list to the user, with what is missing for each.
+4. Set up a skipped datasource only when the user explicitly asks for it:
+   install or authenticate the backing CLI first, then configure it.
+5. E-mail datasources (`gmail`, `mail-export`, `mailcrawl`) matter to most
+   users — always probe them and report their status, even when they end up
+   skipped.
 
 Datasource skills belong in trusted config and remain default-deny. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
@@ -228,10 +256,12 @@ autorag search "summarize the collection" --top-k 3 --json --debug
 ## Keep indexes fresh
 
 For continuous freshness, create or verify an OS-appropriate scheduled
-`autorag watch --once` job, typically every 15–30 minutes. Prefer cron or
-launchd on macOS, cron or a user systemd timer on Linux, and Task Scheduler on
-Windows. Use the same config as search, avoid overlapping runs, and keep logs
-outside source trees.
+`autorag watch --once` job, hourly by default (every 1 hour; shorten only when
+the user asks for fresher indexes). Prefer cron or launchd on macOS, cron or a
+user systemd timer on Linux, and Task Scheduler on Windows. Use the same
+config as search, avoid overlapping runs, and keep logs outside source trees.
+Once the schedule is installed, tell the user right away that hourly freshness
+is set up.
 
 ## Environment overrides
 
@@ -246,6 +276,9 @@ outside source trees.
 ## Completion condition
 
 Setup is complete only when the CLI is installed, roots are approved, a
-non-secret single-model config is written, `status` is acceptable, live `health`
+non-secret single-model config is written, every datasource has been probed
+and the auto-configured and skipped lists reported to the user, dupey is
+installed or its absence reported, `status` is acceptable, live `health`
 passes, `refresh` builds the requested indexes, one real structured search
-succeeds, and any requested ongoing schedule is installed or verified.
+succeeds, and any requested ongoing schedule is installed or verified with the
+user told it is active.