@autorag/librarian 2.0.0 → 2.2.0

package/dist/parser/defaults.d.ts

@@ -1,6 +1,8 @@
+import { type HwpParserOptions } from "./hwp.js";
 import { type OcrParserOptions } from "./ocr.js";
 import { ParserRegistry } from "./registry.js";
 export interface DefaultParserRegistryOptions {
     readonly ocr?: OcrParserOptions;
+    readonly hwp?: HwpParserOptions;
 }
 export declare function createDefaultParserRegistry(options?: DefaultParserRegistryOptions): ParserRegistry;

package/dist/parser/hwp-markdown.d.ts

@@ -0,0 +1,25 @@
+export interface HwpExtractedDocument {
+    readonly paragraphs: readonly HwpBodyParagraph[];
+    readonly tables: readonly HwpTable[];
+}
+export interface HwpBodyParagraph {
+    readonly sectionIndex: number;
+    readonly paragraphIndex: number;
+    readonly text: string;
+}
+export interface HwpTable {
+    readonly sectionIndex: number;
+    readonly parentParagraphIndex: number;
+    readonly controlIndex: number;
+    readonly rowCount: number;
+    readonly columnCount: number;
+    readonly cells: readonly HwpTableCell[];
+}
+export interface HwpTableCell {
+    readonly row: number;
+    readonly column: number;
+    readonly rowSpan: number;
+    readonly columnSpan: number;
+    readonly paragraphs: readonly string[];
+}
+export declare function renderHwpMarkdown(document: HwpExtractedDocument): string;

package/dist/parser/hwp.d.ts

@@ -1,6 +1,13 @@
+import { type HwpExtractionLimits, type HwpExtractor } from "./rhwp-adapter.js";
 import { type ParseInput, type ParseOutput, Parser } from "./types.js";
+export interface HwpParserOptions {
+    readonly limits?: HwpExtractionLimits;
+    readonly extractor?: HwpExtractor;
+}
 export declare class HwpParser extends Parser {
     readonly name = "hwp";
     readonly extensions: readonly [".hwpx", ".hwp"];
+    private readonly extractor;
+    constructor(options?: HwpParserOptions);
     parse(input: ParseInput): Promise<ParseOutput>;
 }

package/dist/parser/index.d.ts

@@ -1,10 +1,11 @@
 export { createDefaultParserRegistry, type DefaultParserRegistryOptions } from "./defaults.js";
 export { EmlParser } from "./eml.js";
 export { ParseError } from "./errors.js";
-export { HwpParser } from "./hwp.js";
+export { HwpParser, type HwpParserOptions } from "./hwp.js";
 export { ImageOcrParser, type OcrEngine, type OcrEngineInput, type OcrParserOptions } from "./ocr.js";
 export { DocxParser, PptxParser, XlsxParser } from "./office.js";
 export { OpendataloaderPdfParser, type PdfConverter } from "./opendataloader-pdf.js";
 export { PlainTextParser } from "./plain-text.js";
 export { ParserRegistry } from "./registry.js";
+export type { HwpExtractionLimits, HwpExtractor } from "./rhwp-adapter.js";
 export { type ParseInput, type ParseOutput, Parser } from "./types.js";

package/dist/parser/rhwp-adapter.d.ts

@@ -0,0 +1,30 @@
+import type { HwpExtractedDocument } from "./hwp-markdown.js";
+export interface HwpExtractionLimits {
+    readonly maxSections?: number;
+    readonly maxBodyParagraphs?: number;
+    readonly maxTables?: number;
+    readonly maxCells?: number;
+    readonly maxCellParagraphs?: number;
+    readonly maxCharacters?: number;
+}
+export type HwpExtractor = (bytes: Uint8Array) => Promise<HwpExtractedDocument>;
+export interface RhwpDocumentApi {
+    free(): void;
+    getCellInfo(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number): string;
+    getCellParagraphCount(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number): number;
+    getCellParagraphLength(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number, cellParagraphIndex: number): number;
+    getPageControlLayout(pageNumber: number): string;
+    getParagraphCount(sectionIndex: number): number;
+    getParagraphLength(sectionIndex: number, paragraphIndex: number): number;
+    getSectionCount(): number;
+    getTableDimensions(sectionIndex: number, parentParagraphIndex: number, controlIndex: number): string;
+    getTextInCell(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number, cellParagraphIndex: number, characterOffset: number, count: number): string;
+    getTextRange(sectionIndex: number, paragraphIndex: number, characterOffset: number, count: number): string;
+    pageCount(): number;
+}
+export interface RhwpRuntime {
+    initialize(): Promise<void>;
+    open(bytes: Uint8Array): RhwpDocumentApi;
+}
+export declare function createRhwpExtractor(runtime?: RhwpRuntime, overrides?: HwpExtractionLimits): HwpExtractor;
+export declare function extractHwpWithRhwp(bytes: Uint8Array, limits?: HwpExtractionLimits): Promise<HwpExtractedDocument>;

package/dist/retrieval/merger.d.ts

@@ -1,6 +1,7 @@
 import type { RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalWithDiagnostics } from "./types.js";
 export interface MergeOptions {
     topK: number;
+    /** Retained for API compatibility; exact-source aggregation is always enabled. */
     dedup: boolean;
 }
 export declare class ResultMerger {

package/dist/retrieval/scope.d.ts

@@ -1,4 +1,14 @@
+export interface RetrievalScopeBinding {
+    readonly virtualPrefix: string;
+    readonly physicalRoots: readonly string[];
+}
+export declare class RetrievalScopeError extends Error {
+    readonly code = "invalid-retrieval-scope";
+    constructor(virtualPrefixes: readonly string[]);
+}
 export declare function normalizeVirtualPath(value: string): string;
 export declare function normalizeVirtualPathScope(scope: string | undefined): string | undefined;
+export declare function resolveRetrievalScope(scope: string | undefined, bindings: readonly RetrievalScopeBinding[], platform?: NodeJS.Platform, passthroughVirtualPrefixes?: readonly string[]): string | undefined;
+export declare function buildRetrievalScopeBindings(workspaceRoot: string, searchPaths: readonly string[], configuredSearchPaths?: readonly string[]): readonly RetrievalScopeBinding[];
 export declare function matchesVirtualPathScope(virtualPath: string, scope: string | undefined): boolean;
 export declare function virtualPathScopeToRegExp(scope: string): RegExp;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -17,7 +17,7 @@
     }
   },
   "bin": {
-    "autorag": "./dist/cli/index.js"
+    "autorag": "dist/cli/index.js"
   },
   "files": [
     "dist",
@@ -39,28 +39,29 @@
     "node": ">=24.0.0"
   },
   "dependencies": {
-    "@earendil-works/pi-agent-core": "^0.80.10",
-    "@earendil-works/pi-ai": "^0.80.10",
-    "@earendil-works/pi-coding-agent": "0.80.10",
-    "@opendataloader/pdf": "^2.4.7",
+    "@earendil-works/pi-agent-core": "^0.84.2",
+    "@earendil-works/pi-ai": "^0.84.2",
+    "@earendil-works/pi-coding-agent": "0.84.2",
+    "@opendataloader/pdf": "^2.5.2",
     "@pngwasi/node-tantivy-binding": "^0.3.4",
+    "@rhwp/core": "0.8.4",
     "chardet": "^2.2.0",
-    "fast-xml-parser": "^5.9.3",
-    "iconv-lite": "^0.7.2",
+    "fast-xml-parser": "^5.11.0",
+    "iconv-lite": "^0.7.3",
     "jszip": "^3.10.1",
-    "mailparser": "^3.9.12",
-    "pi-subagents": "0.35.1",
-    "slides-grab": "^1.3.1",
-    "smol-toml": "1.7.0",
+    "mailparser": "^3.9.15",
+    "pi-subagents": "0.53.0",
+    "slides-grab": "^1.5.0",
+    "smol-toml": "1.8.0",
     "tesseract.js": "^7.0.0",
-    "typebox": "^1.1.24",
-    "yaml": "^2.8.2"
+    "typebox": "^1.3.16",
+    "yaml": "^2.9.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.3.5",
+    "@biomejs/biome": "2.5.9",
     "@types/mailparser": "^3.4.6",
-    "@types/node": "^24.3.0",
-    "typescript": "^5.7.3",
-    "vitest": "^4.1.10"
+    "@types/node": "^26.2.0",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.11"
   }
 }

package/skills/autorag-setup/SKILL.md

@@ -232,6 +232,80 @@ via the role-specific flags or:
 - legacy single-model aliases: `AUTORAG_MODEL_PROVIDER` / `AUTORAG_MODEL_ID`
 - `AUTORAG_SEARCH_PATHS`, `AUTORAG_WORKSPACE`, `AUTORAG_MEMORY_PATH`
 
+## Configure datasource skills (optional)
+
+External datasources (WhatsApp, Telegram, Slack, Discord, Notion, GitHub Issues/PRs, Google
+Drive, Gmail, local mail exports, Obsidian vaults, RSS/news) are configured
+as **datasource skills** in the trusted config file — never via model/tool
+arguments. Add a `datasources` section (skill name → config) plus a trusted
+`datasourceAccess` allow-list to `config.json`:
+
+```jsonc
+{
+  "datasources": {
+    "whatsapp": { "instanceId": "personal", "connector": { "binaryPath": "wacrawl" } },
+    "telegram": { "instanceId": "personal", "connector": { "binaryPath": "telecrawl" } },
+    "slack":    { "connector": { "binaryPath": "slacrawl", "configPath": "/path/to/slacrawl.yaml", "syncSource": "primary" } },
+    "discord":  { "connector": { "tokenEnv": "DISCORD_BOT_TOKEN", "guildId": "..." } },
+    "notion":   { "connector": { "binaryPath": "notcrawl", "configPath": "/path/to/notcrawl.yaml" } },
+    "github":   { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
+    "gdrive":   { "connector": { "backend": "rclone", "remote": "gdrive:" } },
+    "gmail":    { "connector": { "backend": "himalaya", "account": "gmail", "folder": "INBOX" } },
+    "mail-export": { "connector": { "paths": ["/path/to/exports"] } },
+    "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
+    "rss":      { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
+  },
+  "datasourceAccess": {
+    "allowedTags": ["whatsapp", "telegram", "slack", "github", "rss"],
+    "allowedScopes": ["/whatsapp/**", "/telegram/**", "/slack/**", "/github/**", "/rss/**"]
+  }
+}
+```
+
+Rules:
+
+- REST connector tokens are configured as **environment variable names** (`tokenEnv`), never
+  raw secrets in the file. Each skill has a default env var
+  (`DISCORD_BOT_TOKEN`, `GITHUB_TOKEN`,
+  `GDRIVE_ACCESS_TOKEN`, `GMAIL_ACCESS_TOKEN`).
+- **CLI-bridge backends** (recommended where available): `gmail` accepts
+  `"backend": "himalaya"` to index any IMAP/Maildir account the external
+  [himalaya](https://pimalaya.org) CLI has configured (no OAuth plumbing);
+  `gdrive` accepts `"backend": "rclone"` to index a Google Drive remote (or
+  any of rclone's 70+ backends) configured via `rclone config` — Docs/Sheets
+  are exported as text through `--drive-export-formats`. Auth lives entirely
+  in the external tool's own config, matching the katok pattern.
+- `whatsapp` requires the external
+  [wacrawl](https://github.com/openclaw/wacrawl) binary
+  (`brew install openclaw/tap/wacrawl`). Optional trusted connector fields are
+  `binaryPath`, `databasePath`, and `sourcePath`. Live desktop ingestion is
+  macOS-only; the crawler owns Full Disk Access and archive setup.
+- `telegram` requires the external
+  [telecrawl](https://github.com/openclaw/telecrawl) binary
+  (`brew install openclaw/tap/telecrawl`). Optional trusted connector fields
+  are `binaryPath`, `databasePath`, and `sourcePath`. Live desktop ingestion
+  is macOS-only; the crawler owns Full Disk Access and archive setup.
+- `slack` requires the external
+  [slacrawl](https://github.com/openclaw/slacrawl) binary
+  (`brew install openclaw/tap/slacrawl`). Optional trusted connector fields
+  are `binaryPath`, `configPath`, and `syncSource`; Slack credentials and
+  remote source definitions stay in slacrawl's own configuration.
+- `notion` requires the external
+  [notcrawl](https://github.com/openclaw/notcrawl) binary
+  (`brew install openclaw/tap/notcrawl`). Optional trusted connector fields
+  are `binaryPath` and `configPath`; Notion credentials and workspace
+  definitions stay in notcrawl's own configuration.
+- Access is **default-deny**: without `datasourceAccess.allowedTags`, no
+  datasource skill is announced or searchable, even when configured.
+  `allowedScopes` are opaque slash-hierarchical roots like `/slack/<instance>/**`.
+- Per-skill options: `instanceId`, `pollingIntervalMs`, `tags`, and the
+  connector-specific `connector` object. `false` or `{ "enabled": false }`
+  disables an entry. Unknown skill names fail config resolution.
+- `autorag refresh --method datasources` indexes them; chunks persist under
+  `<workspace>/.autorag/datasources/<skill>/<instance>/`.
+- Verify with the manual QA harness: `bun scripts/manual-qa/run-qa.ts`
+  (see `docs/manual-qa-datasources.md`).
+
 ## Build indexes after install
 
 Installation + `init` alone does **not** make search useful. Immediately after