@autorag/librarian 2.0.0 → 2.1.0

package/dist/parser/defaults.d.ts

@@ -1,6 +1,8 @@
+import { type HwpParserOptions } from "./hwp.js";
 import { type OcrParserOptions } from "./ocr.js";
 import { ParserRegistry } from "./registry.js";
 export interface DefaultParserRegistryOptions {
     readonly ocr?: OcrParserOptions;
+    readonly hwp?: HwpParserOptions;
 }
 export declare function createDefaultParserRegistry(options?: DefaultParserRegistryOptions): ParserRegistry;

package/dist/parser/hwp-markdown.d.ts

@@ -0,0 +1,25 @@
+export interface HwpExtractedDocument {
+    readonly paragraphs: readonly HwpBodyParagraph[];
+    readonly tables: readonly HwpTable[];
+}
+export interface HwpBodyParagraph {
+    readonly sectionIndex: number;
+    readonly paragraphIndex: number;
+    readonly text: string;
+}
+export interface HwpTable {
+    readonly sectionIndex: number;
+    readonly parentParagraphIndex: number;
+    readonly controlIndex: number;
+    readonly rowCount: number;
+    readonly columnCount: number;
+    readonly cells: readonly HwpTableCell[];
+}
+export interface HwpTableCell {
+    readonly row: number;
+    readonly column: number;
+    readonly rowSpan: number;
+    readonly columnSpan: number;
+    readonly paragraphs: readonly string[];
+}
+export declare function renderHwpMarkdown(document: HwpExtractedDocument): string;

package/dist/parser/hwp.d.ts

@@ -1,6 +1,13 @@
+import { type HwpExtractionLimits, type HwpExtractor } from "./rhwp-adapter.js";
 import { type ParseInput, type ParseOutput, Parser } from "./types.js";
+export interface HwpParserOptions {
+    readonly limits?: HwpExtractionLimits;
+    readonly extractor?: HwpExtractor;
+}
 export declare class HwpParser extends Parser {
     readonly name = "hwp";
     readonly extensions: readonly [".hwpx", ".hwp"];
+    private readonly extractor;
+    constructor(options?: HwpParserOptions);
     parse(input: ParseInput): Promise<ParseOutput>;
 }

package/dist/parser/index.d.ts

@@ -1,10 +1,11 @@
 export { createDefaultParserRegistry, type DefaultParserRegistryOptions } from "./defaults.js";
 export { EmlParser } from "./eml.js";
 export { ParseError } from "./errors.js";
-export { HwpParser } from "./hwp.js";
+export { HwpParser, type HwpParserOptions } from "./hwp.js";
 export { ImageOcrParser, type OcrEngine, type OcrEngineInput, type OcrParserOptions } from "./ocr.js";
 export { DocxParser, PptxParser, XlsxParser } from "./office.js";
 export { OpendataloaderPdfParser, type PdfConverter } from "./opendataloader-pdf.js";
 export { PlainTextParser } from "./plain-text.js";
 export { ParserRegistry } from "./registry.js";
+export type { HwpExtractionLimits, HwpExtractor } from "./rhwp-adapter.js";
 export { type ParseInput, type ParseOutput, Parser } from "./types.js";

package/dist/parser/rhwp-adapter.d.ts

@@ -0,0 +1,30 @@
+import type { HwpExtractedDocument } from "./hwp-markdown.js";
+export interface HwpExtractionLimits {
+    readonly maxSections?: number;
+    readonly maxBodyParagraphs?: number;
+    readonly maxTables?: number;
+    readonly maxCells?: number;
+    readonly maxCellParagraphs?: number;
+    readonly maxCharacters?: number;
+}
+export type HwpExtractor = (bytes: Uint8Array) => Promise<HwpExtractedDocument>;
+export interface RhwpDocumentApi {
+    free(): void;
+    getCellInfo(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number): string;
+    getCellParagraphCount(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number): number;
+    getCellParagraphLength(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number, cellParagraphIndex: number): number;
+    getPageControlLayout(pageNumber: number): string;
+    getParagraphCount(sectionIndex: number): number;
+    getParagraphLength(sectionIndex: number, paragraphIndex: number): number;
+    getSectionCount(): number;
+    getTableDimensions(sectionIndex: number, parentParagraphIndex: number, controlIndex: number): string;
+    getTextInCell(sectionIndex: number, parentParagraphIndex: number, controlIndex: number, cellIndex: number, cellParagraphIndex: number, characterOffset: number, count: number): string;
+    getTextRange(sectionIndex: number, paragraphIndex: number, characterOffset: number, count: number): string;
+    pageCount(): number;
+}
+export interface RhwpRuntime {
+    initialize(): Promise<void>;
+    open(bytes: Uint8Array): RhwpDocumentApi;
+}
+export declare function createRhwpExtractor(runtime?: RhwpRuntime, overrides?: HwpExtractionLimits): HwpExtractor;
+export declare function extractHwpWithRhwp(bytes: Uint8Array, limits?: HwpExtractionLimits): Promise<HwpExtractedDocument>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -17,7 +17,7 @@
     }
   },
   "bin": {
-    "autorag": "./dist/cli/index.js"
+    "autorag": "dist/cli/index.js"
   },
   "files": [
     "dist",
@@ -44,6 +44,7 @@
     "@earendil-works/pi-coding-agent": "0.80.10",
     "@opendataloader/pdf": "^2.4.7",
     "@pngwasi/node-tantivy-binding": "^0.3.4",
+    "@rhwp/core": "0.7.19",
     "chardet": "^2.2.0",
     "fast-xml-parser": "^5.9.3",
     "iconv-lite": "^0.7.2",

package/skills/autorag-setup/SKILL.md

@@ -232,6 +232,58 @@ via the role-specific flags or:
 - legacy single-model aliases: `AUTORAG_MODEL_PROVIDER` / `AUTORAG_MODEL_ID`
 - `AUTORAG_SEARCH_PATHS`, `AUTORAG_WORKSPACE`, `AUTORAG_MEMORY_PATH`
 
+## Configure datasource skills (optional)
+
+External datasources (Slack, Discord, Notion, GitHub Issues/PRs, Google
+Drive, Gmail, local mail exports, Obsidian vaults, RSS/news) are configured
+as **datasource skills** in the trusted config file — never via model/tool
+arguments. Add a `datasources` section (skill name → config) plus a trusted
+`datasourceAccess` allow-list to `config.json`:
+
+```jsonc
+{
+  "datasources": {
+    "slack":    { "connector": { "tokenEnv": "SLACK_BOT_TOKEN", "channels": ["eng"] } },
+    "discord":  { "connector": { "tokenEnv": "DISCORD_BOT_TOKEN", "guildId": "..." } },
+    "notion":   { "connector": { "tokenEnv": "NOTION_TOKEN" } },
+    "github":   { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
+    "gdrive":   { "connector": { "backend": "rclone", "remote": "gdrive:" } },
+    "gmail":    { "connector": { "backend": "himalaya", "account": "gmail", "folder": "INBOX" } },
+    "mail-export": { "connector": { "paths": ["/path/to/exports"] } },
+    "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
+    "rss":      { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
+  },
+  "datasourceAccess": {
+    "allowedTags": ["slack", "github", "rss"],
+    "allowedScopes": ["/slack/**", "/github/**", "/rss/**"]
+  }
+}
+```
+
+Rules:
+
+- Tokens are configured as **environment variable names** (`tokenEnv`), never
+  raw secrets in the file. Each skill has a default env var
+  (`SLACK_BOT_TOKEN`, `DISCORD_BOT_TOKEN`, `NOTION_TOKEN`, `GITHUB_TOKEN`,
+  `GDRIVE_ACCESS_TOKEN`, `GMAIL_ACCESS_TOKEN`).
+- **CLI-bridge backends** (recommended where available): `gmail` accepts
+  `"backend": "himalaya"` to index any IMAP/Maildir account the external
+  [himalaya](https://pimalaya.org) CLI has configured (no OAuth plumbing);
+  `gdrive` accepts `"backend": "rclone"` to index a Google Drive remote (or
+  any of rclone's 70+ backends) configured via `rclone config` — Docs/Sheets
+  are exported as text through `--drive-export-formats`. Auth lives entirely
+  in the external tool's own config, matching the katok pattern.
+- Access is **default-deny**: without `datasourceAccess.allowedTags`, no
+  datasource skill is announced or searchable, even when configured.
+  `allowedScopes` are opaque slash-hierarchical roots like `/slack/<instance>/**`.
+- Per-skill options: `instanceId`, `pollingIntervalMs`, `tags`, and the
+  connector-specific `connector` object. `false` or `{ "enabled": false }`
+  disables an entry. Unknown skill names fail config resolution.
+- `autorag refresh --method datasources` indexes them; chunks persist under
+  `<workspace>/.autorag/datasources/<skill>/<instance>/`.
+- Verify with the manual QA harness: `bun scripts/manual-qa/run-qa.ts`
+  (see `docs/manual-qa-datasources.md`).
+
 ## Build indexes after install
 
 Installation + `init` alone does **not** make search useful. Immediately after