@autorag/librarian 2.5.3 → 2.6.0

package/NOTICE

@@ -6,22 +6,16 @@ legacy/ package license: Apache-2.0
 
 Third-party production dependencies:
 
-@earendil-works/pi-agent-core@0.85.1
+@earendil-works/pi-agent-core@0.86.1
 License: MIT
 
-@earendil-works/pi-ai@0.85.1
+@earendil-works/pi-ai@0.86.1
 License: MIT
 
-@earendil-works/pi-coding-agent@0.85.1
+@earendil-works/pi-coding-agent@0.86.1
 License: MIT
 
-@earendil-works/pi-tui@0.85.1
-License: MIT
-
-@opendataloader/pdf@2.5.7
-License: Apache-2.0
-
-@rhwp/core@0.8.6
+@earendil-works/pi-tui@0.86.1
 License: MIT
 
 chardet@2.2.0
@@ -36,9 +30,15 @@ License: MIT
 jszip@3.10.1
 License: (MIT OR GPL-3.0-or-later)
 
+kordoc@4.15.7
+License: MIT
+
 mailparser@3.9.20
 License: MIT
 
+pdfjs-dist@6.3.289
+License: Apache-2.0
+
 puppeteer-core@25.11.0
 License: Apache-2.0
 
@@ -54,9 +54,6 @@ License: Apache-2.0
 typebox@1.3.25
 License: MIT
 
-xlsx@0.18.5
-License: Apache-2.0
-
 yaml@2.9.0
 License: ISC
 

package/README.md

@@ -85,6 +85,26 @@ If you are an AI coding agent or LLM (Claude Code, Cursor, Windsurf, Codex, Senp
 | **`autorag-lite-setup`** | [`skills/autorag-lite-setup/`](skills/autorag-lite-setup/SKILL.md) | Initializing and maintaining the model-free AutoRAG Lite lifecycle without an LLM. |
 | **`autorag-lite-search`** | [`skills/autorag-lite-search/`](skills/autorag-lite-search/SKILL.md) | Performing model-free retrieval, reporting evidence, and recording feedback without an LLM. |
 
+### Install the skills into your coding agent
+
+Skills are not auto-discovered — copy the folders you want into the agent's skill directory. For **Claude Code** that directory is `.claude/skills/` in the project (or `~/.claude/skills/` to enable them everywhere). Once `autorag-lite-setup` and `autorag-lite-search` are in place, Claude Code drives `autorag lite retrieve` as a search tool:
+
+```bash
+# From a clone of this repository
+mkdir -p .claude/skills
+cp -R skills/autorag-lite-setup skills/autorag-lite-search .claude/skills/
+```
+
+```bash
+# From a global install — the npm package ships the same skills/ folder
+AUTORAG_SKILLS="$(npm root -g)/@autorag/librarian/skills"
+# Bun global installs live at ~/.bun/install/global/node_modules/@autorag/librarian/skills
+mkdir -p .claude/skills
+cp -R "$AUTORAG_SKILLS/autorag-lite-setup" "$AUTORAG_SKILLS/autorag-lite-search" .claude/skills/
+```
+
+Copy `skills/autorag` and `skills/autorag-setup` the same way when the agent should also drive the model-backed librarian, and `skills/autorag-doctor` for diagnostics. Other agents read their own directories (for example `~/.agents/skills/`) — copy the same folders there and reload the agent session so it picks them up.
+
 ### Quick Agent Workflow
 
 1. **Install CLI:**
@@ -175,7 +195,7 @@ bun add @autorag/librarian
 
 ### System Prerequisites
 
-- **Java 11 or newer:** Required for PDF parsing via `@opendataloader/pdf`. Check with `java -version`.
+- **No Java required:** Document parsing (HWP/HWPX/HWPML, PDF, DOCX, XLSX/XLS) runs in-process through [`kordoc`](https://github.com/chrisryugj/kordoc).
 - **Rust Toolchain (Optional):** Automatically compiles Jikji (`jikji-cli`) if installed.
 - **MinSync:** Automatically downloaded and installed into `<workspace>/.autorag/bin` on first run.
 
@@ -189,6 +209,9 @@ bun add @autorag/librarian
 # Initialize configuration for your documents folder
 autorag init --search-paths ~/Documents/research
 
+# Korean + English is the default; set it explicitly for other corpora
+autorag init --search-paths ~/Documents/research --languages ja,en
+
 # Index documents (parses PDFs/Markdown, builds BM25 and MinSync vectors)
 autorag refresh
 
@@ -196,6 +219,26 @@ autorag refresh
 autorag status
 ```
 
+### Document languages and parsers
+
+One global `languages` setting describes the corpus. It is resolved from
+`--languages`, then `AUTORAG_LANGUAGES`, then `languages` in the config file,
+falling back to `["ko", "en"]`. Supported tags: `ko`, `en`, `ja`, `zh-hans`,
+`zh-hant`, `fr`, `de`, `es`, `ru`, `it`, `pt`, `vi`, `th`, `ar`, `hi`.
+
+| Extension | Parser |
+|---|---|
+| `.hwp` `.hwpx` `.hml` `.hwpml` `.pdf` `.docx` `.xlsx` `.xls` | `kordoc` (nested tables, per-sheet workbooks, no Java) |
+| `.pptx` | built-in PPTX reader |
+| `.eml` | built-in mail reader |
+| `.txt` `.text` `.md` `.markdown` | plain text (CP949/EUC-KR aware) |
+| `.png` `.jpg` `.jpeg` `.bmp` `.tiff` `.webp` | image OCR (opt-in) |
+
+OCR is opt-in and never runs unless enabled, so indexing downloads no model by
+default. When enabled, `languages` selects the recognition languages
+(`ja` → `jpn`, `zh-hans` → `chi_sim`, …) for both standalone images and scanned
+PDF pages.
+
 ### 2. Search from CLI
 
 ```bash
@@ -204,9 +247,6 @@ autorag search "What are our primary Q3 deliverables?"
 
 # Launch the interactive Terminal UI (beta)
 autorag tui
-
-# Open the loopback web management dashboard (127.0.0.1)
-autorag ui
 ```
 
 ### 3. Programmatic Usage (TypeScript API)
@@ -238,13 +278,12 @@ agent.recordFeedbackByNumbers(response.sessionId, [1], [2]);
 
 | Command | Description |
 |---|---|
-| `autorag init` | Initialize `~/.autorag/config.json` with search roots and model settings |
+| `autorag init` | Initialize `~/.autorag/config.json` with search roots, document languages, and model settings |
 | `autorag refresh` | Refresh parsed mirrors, MinSync CDC chunks, datasources, and Jikji |
 | `autorag search "<query>"` | Run the librarian agent to curate structured answers |
 | `autorag status` | Inspect corpus freshness, indexing status, and vector readiness |
 | `autorag health` | Check model provider authentication, token validity, and API reachability |
 | `autorag tui` | Open the interactive librarian terminal UI |
-| `autorag ui` | Open the local loopback web dashboard to configure datasources visually |
 | `autorag duplicates [DIR]` | Read-only scan for exact and near-duplicate document families with `dupey` |
 | `autorag lite ...` | Model-free indexing, retrieval, report generation, and status |
 | `autorag feedback <session>` | Record useful / not-useful feedback by item number |
@@ -288,7 +327,7 @@ AutoRAG Agent stands on the shoulders of fantastic open-source projects:
 - **[Jikji](https://github.com/NomaDamas/jikji)** by [NomaDamas](https://github.com/NomaDamas) — High-performance find-first local document discovery.
 - **[dupey](https://github.com/NomaDamas/dupey)** by [NomaDamas](https://github.com/NomaDamas) — Fast duplicate and near-duplicate document family detection.
 - **Federated CLI Authors:** External datasource tools [`lazykatok`](https://github.com/changeroa/lazykatok), [`discrawl`](https://github.com/openclaw/discrawl), [`mailcrawl`](https://github.com/NomaDamas/mailcrawl), [`wacrawl`](https://github.com/openclaw/wacrawl), [`telecrawl`](https://github.com/openclaw/telecrawl), [`slacrawl`](https://github.com/openclaw/slacrawl), [`notcrawl`](https://github.com/openclaw/notcrawl), [`qmd`](https://github.com/tobi/qmd), and [`rclone`](https://rclone.org).
-- **[OpenDataLoader](https://github.com/opendataloader/opendataloader-pdf)** / **Docling** — Robust PDF and document parsing runtimes.
+- **[kordoc](https://github.com/chrisryugj/kordoc)** by [chrisryugj](https://github.com/chrisryugj) — HWP/HWPX/HWPML, PDF, DOCX and XLSX parsing with nested-table fidelity, used as AutoRAG's default document parser.
 
 ---
 

package/dist/agent/agent.d.ts

@@ -4,16 +4,18 @@ import { type DatasourceAccessContextOptions } from "../datasource/access-contex
 import type { DatasourceIndexResult, DatasourceSkill } from "../datasource/types.js";
 import { type DupeyCliOptions } from "../dupey/index.js";
 import { type JikjiFailureReason, type JikjiOptions } from "../jikji/index.js";
+import { type LanguageTag } from "../language.js";
 import type { ResultFeedback } from "../memory/memory.js";
 import { type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
 import { type ParsedMirrorSyncResult } from "../mirror/sync.js";
 import type { PolicyResolver } from "../p2p/policy-filter.js";
-import type { DefaultParserRegistryOptions } from "../parser/index.js";
+import { type DefaultParserRegistryOptions } from "../parser/index.js";
 import { RetrievalEngine } from "../retrieval/engine.js";
 import { RetrievalMethodRegistry } from "../retrieval/registry.js";
 import type { CuratedResult, RetrievalDiagnostic, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
 import { type ScanDuplicateDocumentsDetails } from "./dupey-tool.js";
 import { type JikjiFindProviderResult } from "./jikji-find-tool.js";
+import { type RefreshProgressCounts, type RefreshProgressPhase } from "./refresh-progress.js";
 import { type SearchDocumentDiagnostic, type SearchDocumentsResponse, type SearchDocumentsStreamEvent } from "./search-documents.js";
 import { type WatcherFactory, type WatchRefreshHandle } from "./watch-refresh.js";
 import { type WebFetchToolOptions } from "./web-fetch-tool.js";
@@ -61,6 +63,17 @@ export interface AutoRAGRefreshStatus {
     readonly stale: boolean;
     readonly diagnostics: readonly SearchDocumentDiagnostic[];
     readonly components: AutoRAGRefreshComponentStatus;
+    readonly progress?: {
+        readonly phase: RefreshProgressPhase;
+        readonly sourceFiles?: {
+            readonly total: number;
+        };
+        readonly parsedCounts?: RefreshProgressCounts;
+        readonly minsync?: {
+            readonly synced?: number;
+        };
+        readonly ownerAlive?: boolean;
+    };
     /** Path-free failure summary of the last refresh, if it failed. */
     readonly lastError?: string;
 }
@@ -110,6 +123,7 @@ export interface AutoRAGAgentOptions {
     apiKey?: string;
     providerApiKeys?: Readonly<Record<string, string>>;
     searchPaths: string[];
+    languages?: readonly LanguageTag[];
     manifestDir?: string;
     memoryPath?: string;
     workspacePath?: string;
@@ -197,6 +211,7 @@ export declare class AutoRAGAgent {
     private refreshState;
     private readonly searchPaths;
     private readonly configuredSearchPaths;
+    readonly languages: readonly LanguageTag[];
     private retrievalScopeBindings;
     private readonly datasourceVirtualScopePrefixes;
     private readonly workspaceProjectRoot;

package/dist/agent/refresh-progress.d.ts

@@ -0,0 +1,30 @@
+export type RefreshProgressPhase = "parsed" | "minsync" | "datasources" | "jikji" | "finalizing";
+export interface RefreshProgressCounts {
+    readonly scanned: number;
+    readonly written: number;
+    readonly deleted: number;
+    readonly skipped: number;
+}
+export interface PersistedRefreshProgress {
+    readonly version: 1;
+    readonly runId: string;
+    readonly pid: number;
+    readonly state: "running" | "success" | "failed";
+    readonly phase: RefreshProgressPhase;
+    readonly startedAt: string;
+    readonly updatedAt: string;
+    readonly finishedAt?: string;
+    readonly sourceFiles?: {
+        readonly total: number;
+    };
+    readonly parsedCounts?: RefreshProgressCounts;
+    readonly minsync?: {
+        readonly synced?: number;
+    };
+    readonly error?: string;
+}
+export declare function readRefreshProgress(root: string): PersistedRefreshProgress | undefined;
+export declare function writeRefreshProgress(root: string, progress: PersistedRefreshProgress): void;
+export declare function updateRefreshProgress(progress: PersistedRefreshProgress, update: Partial<Omit<PersistedRefreshProgress, "version" | "runId" | "pid" | "startedAt" | "updatedAt">>): PersistedRefreshProgress;
+export declare function removeRefreshProgress(root: string): void;
+export declare function isRefreshOwnerAlive(progress: PersistedRefreshProgress): boolean;

package/dist/agent/search-documents.d.ts

@@ -10,7 +10,7 @@ export type SearchDocumentDiagnosticSeverity = "info" | "warning" | "error";
  * (#21/#22) does not require another source-breaking type change; codes not yet
  * emitted are still valid members.
  */
-export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-unavailable" | "minsync-staging-excluded" | "minsync-sync-failed" | "embedder-unavailable" | "embedding-identity-mismatch" | "parser-skipped" | "parser-failed" | "pdf-java-version" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "pdf-hybrid-unavailable" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "missing-final-emit";
+export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-unavailable" | "minsync-staging-excluded" | "minsync-sync-failed" | "embedder-unavailable" | "embedding-identity-mismatch" | "parser-skipped" | "parser-failed" | "parser-warning" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "retrieval-method-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "refresh-failed" | "refresh-interrupted" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "missing-final-emit";
 export interface SearchDocumentDiagnostic {
     readonly code: SearchDocumentDiagnosticCode;
     readonly severity: SearchDocumentDiagnosticSeverity;

package/dist/cli/commands/report.d.ts

@@ -1,3 +1,6 @@
+import { type AutoRAGResultsDetails } from "../../agent/emit-results-tool.js";
 import type { CommandContext } from "./types.js";
+/** Validate a curated report payload against the `emit_autorag_results` contract. */
+export declare function validateReport(value: unknown): AutoRAGResultsDetails;
 /** Persist a structured report submitted by an external curator. */
 export declare function runReport(ctx: CommandContext): Promise<number>;

package/dist/cli/commands/serve.d.ts

@@ -1,11 +1,12 @@
 import type { InjectionClassifierModel } from "../../p2p/injection-classifier.js";
-import { type SimplexPeerServer, startSimplexPeerServer } from "../../p2p/simplex-server.js";
+import { type SimplexPeerServer, startSimplexPeerServer, syncSimplexPeers } from "../../p2p/simplex-server.js";
 import { type SimplexTransport, type StartSimplexOptions } from "../../p2p/simplex-transport.js";
 import { type ResolvedAgentModel, resolveAgentModel } from "../config.js";
 import type { CommandContext } from "./types.js";
 export interface ServeCommandDeps {
     readonly startSimplexPeerServer?: typeof startSimplexPeerServer;
     readonly startSimplexChat?: (options: StartSimplexOptions) => Promise<SimplexTransport>;
+    readonly syncSimplexPeers?: typeof syncSimplexPeers;
     readonly waitUntilStopped?: (server: SimplexPeerServer) => Promise<void>;
     readonly modelResolver?: typeof resolveAgentModel;
 }

package/dist/cli/config.d.ts

@@ -68,14 +68,6 @@ export interface P2pConfig {
     searchTimeoutMs?: number;
     newFilesPublic?: boolean;
 }
-export interface UiConfig {
-    host?: string;
-    port?: number;
-    allowRemote?: boolean;
-    publicOrigin?: string;
-    corsOrigins?: string[];
-    tokenEnv?: string;
-}
 export interface AgentModelConfig {
     /** Provider identity used for auth lookup and Model.provider (e.g. openrouter, fireworks, ollama). */
     provider: string;
@@ -107,6 +99,7 @@ export interface CliConfig {
     searchPaths: string[];
     workspacePath: string;
     memoryPath: string;
+    languages?: string[];
     model?: AgentModelConfig;
     minSync?: MinSyncMethodConfig;
     jikji?: Record<string, unknown> | false;
@@ -122,8 +115,6 @@ export interface CliConfig {
     datasources?: DatasourcesConfig;
     /** Trusted datasource allow-tags/allow-scopes. Absent ⇒ default-deny. */
     datasourceAccess?: DatasourceAccessContextOptions;
-    /** Optional local/deployment settings for the datasource UI. */
-    ui?: UiConfig;
     /** P2P sharing configuration. Disabled by default. */
     p2p?: P2pConfig;
 }

package/dist/cli/index.d.ts

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-declare const COMMANDS: readonly ["init", "setup", "refresh", "status", "search", "feedback", "evidence", "memory", "index", "watch", "health", "duplicates", "tui", "ui", "serve", "p2p", "lite", "models", "gateway"];
+declare const COMMANDS: readonly ["init", "setup", "refresh", "status", "search", "feedback", "evidence", "memory", "index", "watch", "health", "duplicates", "tui", "serve", "p2p", "lite", "models", "gateway"];
 type CommandName = (typeof COMMANDS)[number];
 export type { CommandName };
 interface ParsedArgs {