@autorag/librarian 2.1.0 → 2.2.0

package/dist/retrieval/merger.d.ts

@@ -1,6 +1,7 @@
 import type { RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalWithDiagnostics } from "./types.js";
 export interface MergeOptions {
     topK: number;
+    /** Retained for API compatibility; exact-source aggregation is always enabled. */
     dedup: boolean;
 }
 export declare class ResultMerger {

package/dist/retrieval/scope.d.ts

@@ -1,4 +1,14 @@
+export interface RetrievalScopeBinding {
+    readonly virtualPrefix: string;
+    readonly physicalRoots: readonly string[];
+}
+export declare class RetrievalScopeError extends Error {
+    readonly code = "invalid-retrieval-scope";
+    constructor(virtualPrefixes: readonly string[]);
+}
 export declare function normalizeVirtualPath(value: string): string;
 export declare function normalizeVirtualPathScope(scope: string | undefined): string | undefined;
+export declare function resolveRetrievalScope(scope: string | undefined, bindings: readonly RetrievalScopeBinding[], platform?: NodeJS.Platform, passthroughVirtualPrefixes?: readonly string[]): string | undefined;
+export declare function buildRetrievalScopeBindings(workspaceRoot: string, searchPaths: readonly string[], configuredSearchPaths?: readonly string[]): readonly RetrievalScopeBinding[];
 export declare function matchesVirtualPathScope(virtualPath: string, scope: string | undefined): boolean;
 export declare function virtualPathScopeToRegExp(scope: string): RegExp;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -39,29 +39,29 @@
     "node": ">=24.0.0"
   },
   "dependencies": {
-    "@earendil-works/pi-agent-core": "^0.80.10",
-    "@earendil-works/pi-ai": "^0.80.10",
-    "@earendil-works/pi-coding-agent": "0.80.10",
-    "@opendataloader/pdf": "^2.4.7",
+    "@earendil-works/pi-agent-core": "^0.84.2",
+    "@earendil-works/pi-ai": "^0.84.2",
+    "@earendil-works/pi-coding-agent": "0.84.2",
+    "@opendataloader/pdf": "^2.5.2",
     "@pngwasi/node-tantivy-binding": "^0.3.4",
-    "@rhwp/core": "0.7.19",
+    "@rhwp/core": "0.8.4",
     "chardet": "^2.2.0",
-    "fast-xml-parser": "^5.9.3",
-    "iconv-lite": "^0.7.2",
+    "fast-xml-parser": "^5.11.0",
+    "iconv-lite": "^0.7.3",
     "jszip": "^3.10.1",
-    "mailparser": "^3.9.12",
-    "pi-subagents": "0.35.1",
-    "slides-grab": "^1.3.1",
-    "smol-toml": "1.7.0",
+    "mailparser": "^3.9.15",
+    "pi-subagents": "0.53.0",
+    "slides-grab": "^1.5.0",
+    "smol-toml": "1.8.0",
     "tesseract.js": "^7.0.0",
-    "typebox": "^1.1.24",
-    "yaml": "^2.8.2"
+    "typebox": "^1.3.16",
+    "yaml": "^2.9.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.3.5",
+    "@biomejs/biome": "2.5.9",
     "@types/mailparser": "^3.4.6",
-    "@types/node": "^24.3.0",
-    "typescript": "^5.7.3",
-    "vitest": "^4.1.10"
+    "@types/node": "^26.2.0",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.11"
   }
 }

package/skills/autorag-setup/SKILL.md

@@ -234,7 +234,7 @@ via the role-specific flags or:
 
 ## Configure datasource skills (optional)
 
-External datasources (Slack, Discord, Notion, GitHub Issues/PRs, Google
+External datasources (WhatsApp, Telegram, Slack, Discord, Notion, GitHub Issues/PRs, Google
 Drive, Gmail, local mail exports, Obsidian vaults, RSS/news) are configured
 as **datasource skills** in the trusted config file — never via model/tool
 arguments. Add a `datasources` section (skill name → config) plus a trusted
@@ -243,9 +243,11 @@ arguments. Add a `datasources` section (skill name → config) plus a trusted
 ```jsonc
 {
   "datasources": {
-    "slack":    { "connector": { "tokenEnv": "SLACK_BOT_TOKEN", "channels": ["eng"] } },
+    "whatsapp": { "instanceId": "personal", "connector": { "binaryPath": "wacrawl" } },
+    "telegram": { "instanceId": "personal", "connector": { "binaryPath": "telecrawl" } },
+    "slack":    { "connector": { "binaryPath": "slacrawl", "configPath": "/path/to/slacrawl.yaml", "syncSource": "primary" } },
     "discord":  { "connector": { "tokenEnv": "DISCORD_BOT_TOKEN", "guildId": "..." } },
-    "notion":   { "connector": { "tokenEnv": "NOTION_TOKEN" } },
+    "notion":   { "connector": { "binaryPath": "notcrawl", "configPath": "/path/to/notcrawl.yaml" } },
     "github":   { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
     "gdrive":   { "connector": { "backend": "rclone", "remote": "gdrive:" } },
     "gmail":    { "connector": { "backend": "himalaya", "account": "gmail", "folder": "INBOX" } },
@@ -254,17 +256,17 @@ arguments. Add a `datasources` section (skill name → config) plus a trusted
     "rss":      { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
   },
   "datasourceAccess": {
-    "allowedTags": ["slack", "github", "rss"],
-    "allowedScopes": ["/slack/**", "/github/**", "/rss/**"]
+    "allowedTags": ["whatsapp", "telegram", "slack", "github", "rss"],
+    "allowedScopes": ["/whatsapp/**", "/telegram/**", "/slack/**", "/github/**", "/rss/**"]
   }
 }
 ```
 
 Rules:
 
-- Tokens are configured as **environment variable names** (`tokenEnv`), never
+- REST connector tokens are configured as **environment variable names** (`tokenEnv`), never
   raw secrets in the file. Each skill has a default env var
-  (`SLACK_BOT_TOKEN`, `DISCORD_BOT_TOKEN`, `NOTION_TOKEN`, `GITHUB_TOKEN`,
+  (`DISCORD_BOT_TOKEN`, `GITHUB_TOKEN`,
   `GDRIVE_ACCESS_TOKEN`, `GMAIL_ACCESS_TOKEN`).
 - **CLI-bridge backends** (recommended where available): `gmail` accepts
   `"backend": "himalaya"` to index any IMAP/Maildir account the external
@@ -273,6 +275,26 @@ Rules:
   any of rclone's 70+ backends) configured via `rclone config` — Docs/Sheets
   are exported as text through `--drive-export-formats`. Auth lives entirely
   in the external tool's own config, matching the katok pattern.
+- `whatsapp` requires the external
+  [wacrawl](https://github.com/openclaw/wacrawl) binary
+  (`brew install openclaw/tap/wacrawl`). Optional trusted connector fields are
+  `binaryPath`, `databasePath`, and `sourcePath`. Live desktop ingestion is
+  macOS-only; the crawler owns Full Disk Access and archive setup.
+- `telegram` requires the external
+  [telecrawl](https://github.com/openclaw/telecrawl) binary
+  (`brew install openclaw/tap/telecrawl`). Optional trusted connector fields
+  are `binaryPath`, `databasePath`, and `sourcePath`. Live desktop ingestion
+  is macOS-only; the crawler owns Full Disk Access and archive setup.
+- `slack` requires the external
+  [slacrawl](https://github.com/openclaw/slacrawl) binary
+  (`brew install openclaw/tap/slacrawl`). Optional trusted connector fields
+  are `binaryPath`, `configPath`, and `syncSource`; Slack credentials and
+  remote source definitions stay in slacrawl's own configuration.
+- `notion` requires the external
+  [notcrawl](https://github.com/openclaw/notcrawl) binary
+  (`brew install openclaw/tap/notcrawl`). Optional trusted connector fields
+  are `binaryPath` and `configPath`; Notion credentials and workspace
+  definitions stay in notcrawl's own configuration.
 - Access is **default-deny**: without `datasourceAccess.allowedTags`, no
   datasource skill is announced or searchable, even when configured.
   `allowedScopes` are opaque slash-hierarchical roots like `/slack/<instance>/**`.

package/dist/datasource/skills/discord/connector.d.ts

@@ -1,30 +0,0 @@
-/**
- * Discord guild connector (issue #1305).
- *
- * Fetches text/announcement channel history through the Discord REST API
- * using a trusted, server-configured bot token. Per-channel permission
- * failures degrade to warnings; a guild-level 429 fails the whole fetch as
- * rate-limited. Never throws; all messages stay path/PII-opaque.
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
-export interface DiscordConnectorOptions {
-    /** Discord REST base; override to point at a mock server in tests. */
-    readonly baseUrl?: string;
-    /** Bot token; explicit value wins over {@link tokenEnv}. */
-    readonly token?: string;
-    /** Env var name holding the bot token. Default `DISCORD_BOT_TOKEN`. */
-    readonly tokenEnv?: string;
-    /** Guild (server) id to index. Required trusted configuration. */
-    readonly guildId?: string;
-    /** Restrict indexing to these channel names or ids. */
-    readonly channels?: readonly string[];
-    readonly timeoutMs?: number;
-    readonly fetchImpl?: typeof fetch;
-    readonly maxPages?: number;
-    readonly maxDocuments?: number;
-}
-export declare class DiscordConnector implements DatasourceConnector {
-    private readonly options;
-    constructor(options?: DiscordConnectorOptions);
-    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
-}

package/dist/datasource/skills/discord/index.d.ts

@@ -1,2 +0,0 @@
-export { DiscordConnector, type DiscordConnectorOptions } from "./connector.js";
-export { DISCORD_SKILL_DEFINITION, DiscordSkill, type DiscordSkillOptions } from "./skill.js";

package/dist/datasource/skills/discord/skill.d.ts

@@ -1,20 +0,0 @@
-/**
- * Discord guild datasource skill (issue #1305).
- *
- * Thin composition of DiscordConnector with the shared
- * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
- * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
- * the shared pipeline, opaque `/discord/<instance>/…` sources, and a
- * progressive-disclosure agent-skill manifest.
- */
-import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import { DiscordConnector, type DiscordConnectorOptions } from "./connector.js";
-export declare const DISCORD_SKILL_DEFINITION: ConnectorSkillDefinition;
-export interface DiscordSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
-    /** Trusted connector configuration; a pre-built connector wins. */
-    readonly connector?: DiscordConnector;
-    readonly connectorOptions?: DiscordConnectorOptions;
-}
-export declare class DiscordSkill extends ConnectorDatasourceSkill {
-    constructor(options?: DiscordSkillOptions);
-}

package/dist/datasource/skills/notion/connector.d.ts

@@ -1,26 +0,0 @@
-/**
- * Notion workspace connector (issue #1302).
- *
- * Enumerates pages/databases via the Notion search API using a trusted,
- * server-configured integration token, then pulls one page of block children
- * per page for text content. Per-page block failures degrade to warnings.
- * Never throws; messages stay path/PII-opaque.
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
-export interface NotionConnectorOptions {
-    /** Notion REST base; override to point at a mock server in tests. */
-    readonly baseUrl?: string;
-    /** Integration token; explicit value wins over {@link tokenEnv}. */
-    readonly token?: string;
-    /** Env var name holding the integration token. Default `NOTION_TOKEN`. */
-    readonly tokenEnv?: string;
-    readonly timeoutMs?: number;
-    readonly fetchImpl?: typeof fetch;
-    readonly maxPages?: number;
-    readonly maxDocuments?: number;
-}
-export declare class NotionConnector implements DatasourceConnector {
-    private readonly options;
-    constructor(options?: NotionConnectorOptions);
-    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
-}

package/dist/datasource/skills/notion/skill.d.ts

@@ -1,20 +0,0 @@
-/**
- * Notion workspace datasource skill (issue #1302).
- *
- * Thin composition of NotionConnector with the shared
- * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
- * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
- * the shared pipeline, opaque `/notion/<instance>/…` sources, and a
- * progressive-disclosure agent-skill manifest.
- */
-import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import { NotionConnector, type NotionConnectorOptions } from "./connector.js";
-export declare const NOTION_SKILL_DEFINITION: ConnectorSkillDefinition;
-export interface NotionSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
-    /** Trusted connector configuration; a pre-built connector wins. */
-    readonly connector?: NotionConnector;
-    readonly connectorOptions?: NotionConnectorOptions;
-}
-export declare class NotionSkill extends ConnectorDatasourceSkill {
-    constructor(options?: NotionSkillOptions);
-}

package/dist/datasource/skills/obsidian/connector.d.ts

@@ -1,22 +0,0 @@
-/**
- * Obsidian vault connector (issue #1314).
- *
- * Walks a trusted, server-configured vault directory for markdown notes,
- * extracting frontmatter/inline tags, wiki links, and embeds. No network
- * access. Per-file read failures aggregate into a count-only warning;
- * absolute paths never appear in warnings (vault-relative segments are the
- * vault's own logical hierarchy and are allowed in doc ids).
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
-export interface ObsidianConnectorOptions {
-    /** Absolute path to the vault root. Required trusted configuration. */
-    readonly vaultPath?: string;
-    readonly maxDocuments?: number;
-    /** Files larger than this are skipped. Default 2 MiB. */
-    readonly maxBytesPerFile?: number;
-}
-export declare class ObsidianConnector implements DatasourceConnector {
-    private readonly options;
-    constructor(options?: ObsidianConnectorOptions);
-    fetch(): Promise<ConnectorFetchResult>;
-}

package/dist/datasource/skills/slack/connector.d.ts

@@ -1,28 +0,0 @@
-/**
- * Slack workspace connector (issue #1300).
- *
- * Fetches channel history through the Slack Web API using a trusted,
- * server-configured bot token. Never throws; failures map onto the coarse
- * connector failure union with path/PII-opaque messages. Per-channel
- * failures degrade to warnings so one denied channel cannot fail the fetch.
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
-export interface SlackConnectorOptions {
-    /** Slack Web API base; override to point at a mock server in tests. */
-    readonly baseUrl?: string;
-    /** Bot token; explicit value wins over {@link tokenEnv}. */
-    readonly token?: string;
-    /** Env var name holding the bot token. Default `SLACK_BOT_TOKEN`. */
-    readonly tokenEnv?: string;
-    /** Restrict indexing to these channel names or ids. */
-    readonly channels?: readonly string[];
-    readonly timeoutMs?: number;
-    readonly fetchImpl?: typeof fetch;
-    readonly maxPages?: number;
-    readonly maxDocuments?: number;
-}
-export declare class SlackConnector implements DatasourceConnector {
-    private readonly options;
-    constructor(options?: SlackConnectorOptions);
-    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
-}

package/dist/datasource/skills/slack/skill.d.ts

@@ -1,20 +0,0 @@
-/**
- * Slack workspace datasource skill (issue #1300).
- *
- * Thin composition of SlackConnector with the shared
- * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
- * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
- * the shared pipeline, opaque `/slack/<instance>/…` sources, and a
- * progressive-disclosure agent-skill manifest.
- */
-import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import { SlackConnector, type SlackConnectorOptions } from "./connector.js";
-export declare const SLACK_SKILL_DEFINITION: ConnectorSkillDefinition;
-export interface SlackSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
-    /** Trusted connector configuration; a pre-built connector wins. */
-    readonly connector?: SlackConnector;
-    readonly connectorOptions?: SlackConnectorOptions;
-}
-export declare class SlackSkill extends ConnectorDatasourceSkill {
-    constructor(options?: SlackSkillOptions);
-}