@autorag/librarian 2.0.0 → 2.2.0

package/README.md

@@ -128,7 +128,58 @@ Security defaults are intentionally strict:
 - model/tool arguments never grant datasource tags or scopes;
 - `search_datasource_documents` accepts only `{ query, topK?, scope? }`, and `scope` can only narrow trusted access.
 
-KakaoTalk is the first datasource skill. It uses the external [`katok`](https://github.com/NomaDamas/katok) CLI only — AutoRAG never reads KakaoTalk databases directly. `katok` failures return diagnostics instead of throwing, and remote embedding egress configuration is rejected before the CLI is spawned.
+#### Supported datasources
+
+| Datasource | Skill | Connects via | Notes |
+|---|---|---|---|
+| KakaoTalk | `katok` | external [`katok`](https://github.com/NomaDamas/katok) CLI | first datasource skill; AutoRAG never reads KakaoTalk databases directly |
+| WhatsApp | `whatsapp` | external [`wacrawl`](https://github.com/openclaw/wacrawl) CLI | local-first incremental archive + FTS5 search; live desktop ingestion is macOS-only |
+| Telegram | `telegram` | external [`telecrawl`](https://github.com/openclaw/telecrawl) CLI | local-first archive + FTS5 search; live desktop ingestion is macOS-only |
+| Slack | `slack` | external [`slacrawl`](https://github.com/openclaw/slacrawl) CLI | local-first workspace/channel/thread archive + FTS5 search |
+| Discord | `discord` | external [`discrawl`](https://github.com/openclaw/discrawl) CLI | guild/channel/thread/DM archive; FTS5 + semantic + hybrid retrieval, incremental sync |
+| Notion | `notion` | external [`notcrawl`](https://github.com/openclaw/notcrawl) CLI | local-first page/database/block archive + FTS5 search |
+| GitHub Issues/PRs | `github` | GitHub REST (token optional) | issues + PR bodies per `owner/repo`; public repos work unauthenticated |
+| Google Drive | `gdrive` | Drive REST v3, or **[`rclone`](https://rclone.org) CLI** (`backend: "rclone"`) | Docs/Sheets exported as text; the rclone backend also opens any of rclone's 70+ remotes |
+| Gmail / IMAP | `gmail` | Gmail REST v1, or **[`himalaya`](https://pimalaya.org) CLI** (`backend: "himalaya"`) | the himalaya backend indexes any IMAP/Maildir account it has configured — no OAuth plumbing |
+| Local mail exports | `mail-export` | filesystem (`.mbox` / `.eml`) | classic `From_` splitting, mailparser-based; count-only warnings |
+| Obsidian vault | `obsidian` | external [`qmd`](https://github.com/tobi/qmd) CLI | incremental `qmd update`, BM25 `qmd search`, semantic `qmd vsearch`; vault path via `connector.vaultPath` |
+| RSS / news | `rss` | HTTP feed polling | RSS 2.0 + Atom, feed/category hierarchy, 24h dedupe window |
+
+Connector-backed skills fetch documents into AutoRAG's local chunk store. External-crawler skills such as KakaoTalk, WhatsApp, Telegram, Slack, and Notion leave incremental archive and FTS ownership with their CLI and map query results into the same retrieval pipeline. Obsidian uses the external `qmd` CLI (incremental update + BM25 + semantic). Tokens are referenced by environment variable name only, never stored in config. Process/API failures surface as path/PII-opaque diagnostics. See [docs/manual-qa-datasources.md](docs/manual-qa-datasources.md) for the QA harnesses.
+
+Configure them in `config.json` (CLI) or pass `datasourceSkills` programmatically:
+
+```jsonc
+{
+  "datasources": {
+    "whatsapp": { "instanceId": "personal", "connector": { "binaryPath": "wacrawl" } },
+    "telegram": { "instanceId": "personal", "connector": { "binaryPath": "telecrawl" } },
+    "slack":    { "connector": { "binaryPath": "slacrawl", "configPath": "/path/to/slacrawl.yaml", "syncSource": "primary" } },
+    "notion":   { "connector": { "binaryPath": "notcrawl", "configPath": "/path/to/notcrawl.yaml" } },
+    "github":   { "connector": { "repos": ["owner/repo"] } },
+    "gmail":    { "connector": { "backend": "himalaya", "account": "gmail", "folder": "INBOX" } },
+    "gdrive":   { "connector": { "backend": "rclone", "remote": "gdrive:" } },
+    "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
+    "rss":      { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
+  },
+  "datasourceAccess": {
+    "allowedTags": ["whatsapp", "telegram", "slack", "notion", "github", "gmail", "gdrive", "obsidian", "rss"],
+    "allowedScopes": ["/whatsapp/**", "/telegram/**", "/slack/**", "/notion/**", "/github/**", "/gmail/**", "/gdrive/**", "/obsidian/**", "/rss/**"]
+  }
+}
+```
+
+Install wacrawl with `brew install openclaw/tap/wacrawl`. AutoRAG invokes `wacrawl sync` during datasource refresh and `wacrawl --json --sync never search` during retrieval. Optional trusted connector fields are `binaryPath`, `databasePath`, and `sourcePath`. The child process receives only a restricted environment; unrelated model/provider secrets are not forwarded. Live WhatsApp Desktop discovery requires macOS and the permissions documented by wacrawl, while an existing portable archive can be queried on other supported platforms.
+
+Install telecrawl with `brew install openclaw/tap/telecrawl`. AutoRAG invokes `telecrawl import` during datasource refresh and `telecrawl --json search` during retrieval. It uses the same optional trusted connector fields and restricted child environment as wacrawl. Live Telegram Desktop discovery requires macOS and the permissions documented by telecrawl, while an existing portable archive can be queried on other supported platforms.
+
+Install slacrawl with `brew install openclaw/tap/slacrawl`. AutoRAG invokes `slacrawl sync` during datasource refresh and `slacrawl --json search` during retrieval. Optional trusted connector fields are `binaryPath`, `configPath`, and `syncSource`. Slack credentials and source definitions remain in slacrawl's own configuration rather than AutoRAG.
+
+Install notcrawl with `brew install openclaw/tap/notcrawl`. AutoRAG invokes `notcrawl sync` during datasource refresh and `notcrawl search --json` during retrieval. Optional trusted connector fields are `binaryPath` and `configPath`. Notion credentials and workspace definitions remain in notcrawl's own configuration rather than AutoRAG.
+
+#### KakaoTalk (katok)
+
+KakaoTalk was the first datasource skill. It uses the external [`katok`](https://github.com/NomaDamas/katok) CLI only — AutoRAG never reads KakaoTalk databases directly. `katok` failures return diagnostics instead of throwing, and remote embedding egress configuration is rejected before the CLI is spawned.
 
 ```typescript
 import { AutoRAGAgent, KatokSkill } from "@autorag/librarian";
@@ -152,6 +203,57 @@ await agent.refresh(); // refreshes parsed mirrors, BM25/MinSync, and datasource
 const results = await agent.searchDatasourceDocuments("meeting with Mina", { topK: 5 });
 ```
 
+#### Discord (discrawl)
+
+Discord uses the external [`discrawl`](https://github.com/openclaw/discrawl) CLI, which owns the SQLite archive, the FTS5 index, and the message vectors. AutoRAG never calls the Discord API itself.
+
+```bash
+brew install openclaw/tap/discrawl
+```
+
+Two archive sources are supported. `wiretap` (the default) reads the local Discord Desktop cache and needs **no token at all**; `discord` uses a bot token, which is the ToS-sanctioned automation path. AutoRAG refuses to spawn the CLI when a Discord *user* token is present in the environment — automating a user account violates Discord's Community Guidelines and can get the account terminated.
+
+```typescript
+import { AutoRAGAgent, DiscrawlClient, DiscrawlSkill } from "@autorag/librarian";
+
+const discord = new DiscrawlSkill({
+  client: new DiscrawlClient({ source: "wiretap", root: process.cwd() }),
+  instanceId: "community",
+});
+
+const agent = new AutoRAGAgent({
+  searchPaths: ["/path/to/documents"],
+  datasourceSkills: [discord],
+  datasourceAccess: {
+    allowedTags: ["discord"],
+    allowedScopes: ["/discord/community/**"],
+  },
+});
+```
+
+Or through the trusted config factory:
+
+```json
+{
+  "datasources": {
+    "discord": {
+      "instanceId": "community",
+      "connector": {
+        "source": "wiretap",
+        "embeddingProvider": "ollama",
+        "embeddingModel": "embeddinggemma",
+        "defaultMode": "hybrid"
+      }
+    }
+  }
+}
+```
+
+Two defaults are deliberate and worth keeping:
+
+- **`defaultMode: "hybrid"`** — discrawl's FTS index strips newlines without substituting a space, welding words across line breaks into a single unsearchable token (measured at ~47% of post-newline words on a real archive). Semantic recall covers that gap. See [#1413](https://github.com/Marker-Inc-Korea/AutoRAG/issues/1413).
+- **`embeddingProvider: "ollama"` + `embeddingModel: "embeddinggemma"`** — semantic search requires an embedding provider (`ollama serve && ollama pull embeddinggemma`). For workspace-managed discrawl state, AutoRAG writes these values to `.autorag/datasources/discrawl/config.toml` while preserving unrelated discrawl settings. An explicit `connector.configPath` remains operator-owned and is never rewritten. EmbeddingGemma (Gemma 3 300M, 768-dim, 100+ languages) is the same model family katok uses for KakaoTalk, so all CLI-backed datasources share one local embedder. Do **not** use `nomic-embed-text`: it is English-only and collapses non-English text into one narrow similarity band, silently degrading semantic search to noise. AutoRAG emits a diagnostic when an English-only model is configured. See [#1414](https://github.com/Marker-Inc-Korea/AutoRAG/issues/1414).
+
 A datasource skill should provide polling/cron metadata for routine indexing, source descriptions for the agent prompt, slash-hierarchical opaque source paths such as `/kakao/personal/chunks/<chunk-id>`, and permission tags that match your server-side access policy.
 
 ### Primary target: document collections
@@ -213,7 +315,7 @@ bun add github:NomaDamas/AutoRAG-2.0
 ```
 
 Git-based installs build `dist/` via the `prepare` script and require Bun on the installing machine.
-External tool binaries auto-install on first use into `<workspace>/.autorag/bin`: **MinSync** downloads a verified GitHub release asset (on by default; `minSync.autoInstall: false` to opt out), and **Jikji** compiles the [`jikji-cli`](https://crates.io/crates/jikji-cli) crate via cargo (requires the [Rust toolchain](https://rustup.rs); `jikji.autoInstall: false` to opt out). New `autorag init` configs enable Jikji find-first discovery by default. KakaoTalk (`katok`) stays a manual, optional install. All of them degrade gracefully when missing — core BM25 search works without any of them.
+External tool binaries auto-install on first use into `<workspace>/.autorag/bin`: **MinSync** downloads a verified GitHub release asset (on by default; `minSync.autoInstall: false` to opt out), and **Jikji** compiles the [`jikji-cli`](https://crates.io/crates/jikji-cli) crate via cargo (requires the [Rust toolchain](https://rustup.rs); `jikji.autoInstall: false` to opt out). New `autorag init` configs enable Jikji find-first discovery by default. KakaoTalk (`katok`) and Discord (`discrawl`) stay manual, optional installs. All of them degrade gracefully when missing — core BM25 search works without any of them.
 
 ## Quick Start
 

package/dist/agent/agent.d.ts

@@ -24,8 +24,14 @@ export interface AutoRAGRefreshOptions {
     /** Restrict refresh to specific methods. Defaults to all when undefined. */
     readonly methods?: readonly RefreshMethod[];
 }
+export interface AutoRAGMinSyncRefreshResult {
+    readonly ok: boolean;
+    readonly synced: number;
+    readonly reason?: string;
+}
 export interface AutoRAGRefreshResult extends ParsedMirrorSyncResult {
     readonly bm25?: BM25SyncResult;
+    readonly minsync?: AutoRAGMinSyncRefreshResult;
     readonly datasources?: readonly DatasourceIndexResult[];
 }
 export interface AutoRAGRefreshComponentStatus {
@@ -122,6 +128,9 @@ export declare class AutoRAGAgent {
     private refreshing;
     private refreshState;
     private readonly searchPaths;
+    private readonly configuredSearchPaths;
+    private retrievalScopeBindings;
+    private readonly datasourceVirtualScopePrefixes;
     private readonly allowedExplorerRoots;
     private readonly workspaceProjectRoot;
     private readonly methodRegistry;
@@ -291,4 +300,6 @@ export declare class AutoRAGAgent {
     /** The retrieval method registry (posix active; vector/bm25/hybrid pluggable). */
     getMethodRegistry(): RetrievalMethodRegistry;
     getSystemPrompt(): string;
+    private normalizeRetrievalOptions;
+    private resolveRetrievalScope;
 }

package/dist/agent/search-bm25-tool.d.ts

@@ -14,5 +14,5 @@ export interface SearchBM25DocumentsDetails {
     readonly readiness: BM25Status["readiness"];
     readonly engine: BM25Status["engine"];
 }
-export declare function createSearchBM25DocumentsTool(getMethod: () => BM25Method | undefined): AgentTool<typeof searchBM25Schema, SearchBM25DocumentsDetails>;
+export declare function createSearchBM25DocumentsTool(getMethod: () => BM25Method | undefined, resolveScope?: (scope: string | undefined) => string | undefined): AgentTool<typeof searchBM25Schema, SearchBM25DocumentsDetails>;
 export {};

package/dist/agent/search-documents.d.ts

@@ -28,6 +28,7 @@ export interface SearchDocumentResult {
     readonly evidence: readonly SearchDocumentEvidence[];
     readonly confidence: number;
     readonly feedbackId: string;
+    readonly source?: string;
 }
 export interface SearchDocumentsResponse {
     readonly sessionId: string;

package/dist/agent/search-minsync-tool.d.ts

@@ -20,5 +20,5 @@ export interface SearchMinSyncDocumentsDetails {
  * tool returns a path-free unavailable message with `available: false` and a
  * zero result count — never a real binary or workspace path.
  */
-export declare function createSearchMinSyncDocumentsTool(getMethod: () => MinSyncVectorMethod | undefined): AgentTool<typeof searchMinSyncSchema, SearchMinSyncDocumentsDetails>;
+export declare function createSearchMinSyncDocumentsTool(getMethod: () => MinSyncVectorMethod | undefined, resolveScope?: (scope: string | undefined) => string | undefined): AgentTool<typeof searchMinSyncSchema, SearchMinSyncDocumentsDetails>;
 export {};

package/dist/cli/config.d.ts

@@ -1,5 +1,7 @@
 import type { Api, Model } from "@earendil-works/pi-ai";
 import type { AutoRAGAgentOptions } from "../agent/agent.js";
+import type { DatasourceAccessContextOptions } from "../datasource/access-context.js";
+import { type DatasourcesConfig } from "../datasource/skills/factory.js";
 import type { EnsureMinSyncBinaryOptions, MinSyncEmbedderConfig } from "../minsync/index.js";
 import type { BM25Engine, BM25FallbackMode } from "../retrieval/methods/bm25.js";
 import { type LoadLocalAutoRAGModelsOptions } from "../subagents/local-models.js";
@@ -84,6 +86,10 @@ export interface CliConfig {
     bm25?: Bm25MethodConfig;
     jikji?: Record<string, unknown>;
     parserOptions?: Record<string, unknown>;
+    /** Trusted datasource skill configuration (skill name → config). */
+    datasources?: DatasourcesConfig;
+    /** Trusted datasource allow-tags/allow-scopes. Absent ⇒ default-deny. */
+    datasourceAccess?: DatasourceAccessContextOptions;
 }
 export interface ResolveConfigInput {
     flags: Record<string, string | boolean | undefined>;