@autorag/librarian 2.5.2 → 2.6.0

package/dist/language.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Global language configuration for AutoRAG.
+ *
+ * The tag set is curated rather than free-form BCP-47: every accepted tag must
+ * map to a concrete OCR engine configuration (see src/parser/ocr-engines.ts).
+ * A tag we cannot map would be a promise the parser stack cannot keep.
+ */
+export declare const AUTORAG_LANGUAGE_TAGS: readonly ["ko", "en", "ja", "zh-hans", "zh-hant", "fr", "de", "es", "ru", "it", "pt", "vi", "th", "ar", "hi"];
+export type LanguageTag = (typeof AUTORAG_LANGUAGE_TAGS)[number];
+/** Korean + English: AutoRAG's primary corpus languages. */
+export declare const DEFAULT_LANGUAGES: readonly LanguageTag[];
+export declare class LanguageError extends Error {
+    readonly name = "LanguageError";
+}
+/**
+ * Normalizes a configured language value into a non-empty, deduplicated,
+ * lowercase tag list. The first entry is the primary language.
+ */
+export declare function normalizeLanguages(raw: unknown): LanguageTag[];
+/** Parses a comma-separated CLI flag or environment variable value. */
+export declare function parseLanguageList(value: string): LanguageTag[];

package/dist/memory/memory.d.ts

@@ -7,7 +7,6 @@ export interface SignalDefaults {
     readonly followupWeight: number;
     readonly retryWeight: number;
     readonly implicitCap: number;
-    readonly decayHalfLifeMs?: number;
 }
 export interface EvidenceContext {
     readonly retrieverMix?: readonly string[];

package/dist/mirror/paths.d.ts

@@ -2,7 +2,9 @@ export declare const PARSED_MIRROR_SUBDIR: string;
 export declare const PARSED_FILES_SUBDIR = "files";
 export declare const PARSED_INDEX_FILE = "index.json";
 export declare const REFRESH_READINESS_FILE = "refresh-complete.json";
+export declare const REFRESH_PROGRESS_FILE = "refresh-progress.json";
 export declare function parsedMirrorRoot(root: string): string;
 export declare function parsedMirrorIndexPath(root: string): string;
 export declare function parsedOutputPath(root: string, virtualPath: string): string;
 export declare function refreshReadinessPath(root: string): string;
+export declare function refreshProgressPath(root: string): string;

package/dist/mirror/sync.d.ts

@@ -13,7 +13,7 @@ export interface ParsedMirrorSyncOptions {
     /** Absolute source paths omitted from parsed mirrors (for example exact duplicate copies). */
     readonly excludeSourcePaths?: ReadonlySet<string>;
 }
-export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "pdf-java-version" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "pdf-hybrid-unavailable";
+export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning";
 /** Path-opaque refresh diagnostic. `source` is an opaque virtual path, never a real fs path. */
 export interface ParsedMirrorDiagnostic {
     readonly code: ParsedMirrorDiagnosticCode;

package/dist/p2p/simplex-server.d.ts

@@ -4,7 +4,7 @@ import type { RetrievalOptions } from "../retrieval/types.js";
 import { type InjectionClassifierModel } from "./injection-classifier.js";
 import { type PolicyQuotas, type PolicyResolution } from "./policy.js";
 import type { PolicyResolver } from "./policy-filter.js";
-import type { SimplexTransport } from "./simplex-transport.js";
+import { type SimplexContact, type SimplexPeerProfile, type SimplexTransport } from "./simplex-transport.js";
 import { type PeerQueryRequest, type PeerQueryResponse } from "./wire.js";
 /**
  * SimpleX message bodies have no documented byte cap, but the wire envelope
@@ -22,11 +22,15 @@ export interface SimplexPeerRecord {
     /** SimpleX contact id of the peer (stable per profile). */
     readonly contactId: number;
     readonly addedAt: string;
-    readonly displayName?: string;
+    /**
+     * The profile this peer shared over SimpleX. It is the source of truth for
+     * who the contact is; only `description` below is added locally.
+     */
+    readonly profile?: SimplexPeerProfile;
+    /** When `profile` was last read from simplex-chat. */
+    readonly profileSyncedAt?: string;
+    /** Your own note about this contact. */
     readonly description?: string;
-    readonly role?: string;
-    readonly org?: string;
-    readonly accessHint?: readonly string[];
 }
 export type SimplexPeerRegistry = Record<string, SimplexPeerRecord>;
 export interface PeerTargetMatch {
@@ -40,6 +44,18 @@ export declare function rankSimplexPeerTargets(query: string, registry: SimplexP
 export declare function loadSimplexPeerRegistry(workspacePath: string): SimplexPeerRegistry;
 /** Persist the SimpleX peer registry. */
 export declare function saveSimplexPeerRegistry(workspacePath: string, registry: SimplexPeerRegistry): void;
+export interface SimplexPeerSyncResult {
+    /** Registry keys whose stored SimpleX profile was refreshed. */
+    readonly updated: readonly string[];
+    /** SimpleX contacts with no trusted record. Reported, never added automatically. */
+    readonly untrusted: readonly SimplexContact[];
+}
+/**
+ * Merge the profiles SimpleX delivered into the records you already trusted.
+ * Contacts that are not in the registry are reported, never trusted: the
+ * registry stays the allowlist for inbound peer queries.
+ */
+export declare function syncSimplexPeers(workspacePath: string, contacts: readonly SimplexContact[], syncedAt?: string): SimplexPeerSyncResult;
 export interface P2pSearchAgent {
     readonly remoteSession: boolean;
     searchDocuments(query: string, options?: RetrievalOptions): Promise<SearchDocumentsResponse>;

package/dist/p2p/simplex-transport.d.ts

@@ -9,6 +9,24 @@ export interface SimplexIncomingMessage {
     readonly text: string;
     readonly chatItemId: number;
 }
+/**
+ * The profile a peer publishes over SimpleX. It is the identity AutoRAG stores
+ * for the contact; only your local name and note are added on top of it.
+ */
+export interface SimplexPeerProfile {
+    readonly displayName: string;
+    readonly fullName?: string;
+    readonly shortDescr?: string;
+    readonly description?: string;
+    readonly image?: string;
+}
+/** A SimpleX contact: the peer's own profile plus the name you call it locally. */
+export interface SimplexContact {
+    readonly contactId: number;
+    readonly localDisplayName: string;
+    /** The profile the peer shared; absent until the peer publishes one. */
+    readonly profile?: SimplexPeerProfile;
+}
 export interface SimplexTransport {
     readonly dbPrefix: string;
     readonly displayName: string;
@@ -16,10 +34,7 @@ export interface SimplexTransport {
     getOrCreateAddress(): Promise<string>;
     createInvitation(): Promise<string>;
     connect(link: string): Promise<void>;
-    listContacts(): Promise<{
-        contactId: number;
-        localDisplayName: string;
-    }[]>;
+    listContacts(): Promise<SimplexContact[]>;
     sendMessage(contactId: number, text: string): Promise<void>;
     onMessage(handler: (message: SimplexIncomingMessage) => void): () => void;
     close(): Promise<void>;
@@ -31,6 +46,8 @@ export interface StartSimplexOptions {
     readonly port?: number;
     readonly connectTimeoutMs?: number;
 }
+/** Extract the stored subset of a SimpleX `LocalProfile`; SimpleX-internal fields are dropped. */
+export declare function parseSimplexProfile(value: unknown): SimplexPeerProfile | undefined;
 /**
  * Spawn `simplex-chat -p <port> -d <dbPrefix>` and connect to its WebSocket
  * bot API. Creates the local user profile if it does not exist yet.

package/dist/parser/defaults.d.ts

@@ -1,10 +1,16 @@
-import { type HwpParserOptions } from "./hwp.js";
+import { type LanguageTag } from "../language.js";
+import { type KordocParserOptions } from "./kordoc.js";
 import { type OcrParserOptions } from "./ocr.js";
-import { type OpendataloaderPdfParserOptions } from "./opendataloader-pdf.js";
 import { ParserRegistry } from "./registry.js";
 export interface DefaultParserRegistryOptions {
     readonly ocr?: OcrParserOptions;
-    readonly hwp?: HwpParserOptions;
-    readonly thinExtract?: OpendataloaderPdfParserOptions["thinExtract"];
+    readonly kordoc?: KordocParserOptions;
+    /** Global document languages; drives OCR engine selection. */
+    readonly languages?: readonly LanguageTag[];
 }
+/**
+ * Merges the globally configured document languages into registry options.
+ * An explicit `parserOptions.languages` wins over the global setting.
+ */
+export declare function resolveParserOptions(parserOptions: DefaultParserRegistryOptions | undefined, languages: readonly LanguageTag[]): DefaultParserRegistryOptions;
 export declare function createDefaultParserRegistry(options?: DefaultParserRegistryOptions): ParserRegistry;

package/dist/parser/index.d.ts

@@ -1,11 +1,9 @@
-export { createDefaultParserRegistry, type DefaultParserRegistryOptions } from "./defaults.js";
+export { createDefaultParserRegistry, type DefaultParserRegistryOptions, resolveParserOptions, } from "./defaults.js";
 export { EmlParser } from "./eml.js";
 export { ParseError } from "./errors.js";
-export { HwpParser, type HwpParserOptions } from "./hwp.js";
+export { KORDOC_EXTENSIONS, type KordocOcrProvider, type KordocParseFn, KordocParser, type KordocParserOptions, } from "./kordoc.js";
 export { ImageOcrParser, type OcrEngine, type OcrEngineInput, type OcrParserOptions } from "./ocr.js";
-export { DocxParser, PptxParser, XlsxParser } from "./office.js";
-export { OpendataloaderPdfParser, type OpendataloaderPdfParserOptions, type PdfConverter, } from "./opendataloader-pdf.js";
+export { PptxParser } from "./office.js";
 export { PlainTextParser } from "./plain-text.js";
 export { ParserRegistry } from "./registry.js";
-export type { HwpExtractionLimits, HwpExtractor } from "./rhwp-adapter.js";
 export { type ParseDiagnostic, type ParseDiagnosticCode, type ParseInput, type ParseOutput, Parser, } from "./types.js";

package/dist/parser/kordoc.d.ts

@@ -0,0 +1,53 @@
+import { type ParseInput, type ParseOutput, Parser } from "./types.js";
+/** Image mime types kordoc hands to an injected OCR provider. */
+export type KordocOcrMimeType = "image/png" | "image/jpeg" | "image/webp";
+/**
+ * OCR seam kordoc calls per page that needs recognition. AutoRAG injects a
+ * tesseract-backed provider so recognition follows the configured languages
+ * instead of kordoc's Korean+Latin built-in model.
+ */
+export type KordocOcrProvider = (pageImage: Uint8Array, pageNumber: number, mimeType: KordocOcrMimeType) => Promise<string>;
+export interface KordocWarning {
+    readonly code?: string;
+    readonly message: string;
+}
+export interface KordocParseSuccess {
+    readonly success: true;
+    readonly fileType?: string;
+    readonly markdown: string;
+    readonly blocks?: readonly unknown[];
+    readonly warnings?: readonly KordocWarning[];
+}
+export interface KordocParseFailure {
+    readonly success: false;
+    readonly fileType?: string;
+    readonly error: string;
+    readonly code?: string;
+}
+export type KordocParseResult = KordocParseSuccess | KordocParseFailure;
+export interface KordocParseCallOptions {
+    readonly ocr?: boolean | "force" | KordocOcrProvider;
+}
+/** Injection seam: the real kordoc `parse`, or a stub in tests. */
+export type KordocParseFn = (bytes: Uint8Array, options: KordocParseCallOptions) => Promise<KordocParseResult>;
+export interface KordocParserOptions {
+    /** OCR mode handed to kordoc: off, built-in engine, or an injected provider. */
+    readonly ocr?: boolean | "force" | KordocOcrProvider;
+    readonly parse?: KordocParseFn;
+}
+/** Document extensions kordoc parses. `.pptx` and `.eml` stay on their own parsers. */
+export declare const KORDOC_EXTENSIONS: readonly [".hwp", ".hwpx", ".hml", ".hwpml", ".pdf", ".docx", ".xlsx", ".xls"];
+/**
+ * Parses HWP/HWPX/HWPML, PDF, DOCX and XLSX/XLS through the kordoc library.
+ *
+ * kordoc reports failures as a result object; AutoRAG's contract is a thrown
+ * `ParseError`, so the code and message are forwarded verbatim (AGENTS.md error
+ * transparency) and warnings become `parser-warning` diagnostics.
+ */
+export declare class KordocParser extends Parser {
+    #private;
+    readonly name = "kordoc";
+    readonly extensions: readonly [".hwp", ".hwpx", ".hml", ".hwpml", ".pdf", ".docx", ".xlsx", ".xls"];
+    constructor(options?: KordocParserOptions);
+    parse(input: ParseInput): Promise<ParseOutput>;
+}

package/dist/parser/ocr-engines.d.ts

@@ -0,0 +1,12 @@
+import type { LanguageTag } from "../language.js";
+export type OcrProvider = (pageImage: Uint8Array, pageNumber: number, mimeType: "image/png" | "image/jpeg" | "image/webp") => Promise<string>;
+export type TesseractOcrEngine = (language: string, pageImage: Uint8Array, pageNumber: number, mimeType: "image/png" | "image/jpeg" | "image/webp") => Promise<string>;
+export interface TesseractOcrProviderOptions {
+    readonly languages: readonly LanguageTag[];
+    readonly timeoutMs?: number;
+    readonly engine?: TesseractOcrEngine;
+}
+/** Traineddata codes for the configured languages, order preserved and deduplicated. */
+export declare function tesseractLanguageListFor(tags: readonly LanguageTag[]): string[];
+export declare function tesseractLanguageFor(tags: readonly LanguageTag[]): string;
+export declare function createTesseractOcrProvider(options: TesseractOcrProviderOptions): OcrProvider;

package/dist/parser/ocr.d.ts

@@ -6,6 +6,10 @@ export interface OcrEngineInput {
     readonly signal: AbortSignal;
 }
 export type OcrEngine = (input: OcrEngineInput) => Promise<string>;
+export type OcrOperation = {
+    readonly result: Promise<string>;
+    readonly cleanup: Promise<void>;
+};
 export interface OcrParserOptions {
     readonly enabled: boolean;
     readonly languages?: readonly string[];
@@ -15,7 +19,7 @@ export interface OcrParserOptions {
 }
 export declare class ImageOcrParser extends Parser {
     readonly name = "image-ocr";
-    readonly extensions: readonly [".jpg", ".jpeg", ".png", ".bmp", ".tiff"];
+    readonly extensions: readonly [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".webp"];
     private readonly languages;
     private readonly timeoutMs;
     private readonly maxBytes;
@@ -23,3 +27,5 @@ export declare class ImageOcrParser extends Parser {
     constructor(options: OcrParserOptions);
     parse(input: ParseInput): Promise<ParseOutput>;
 }
+export declare function createTesseractOcrOperation(input: OcrEngineInput): OcrOperation;
+export declare function withTimeout(operation: OcrOperation, timeoutMs: number, onTimeout: () => void): Promise<string>;

package/dist/parser/office.d.ts

@@ -1,16 +1,7 @@
 import { type ParseInput, type ParseOutput, Parser } from "./types.js";
-export declare class DocxParser extends Parser {
-    readonly name = "docx";
-    readonly extensions: readonly [".docx"];
-    parse(input: ParseInput): Promise<ParseOutput>;
-}
+/** PPTX stays on AutoRAG's own reader: kordoc detects pptx but does not parse it. */
 export declare class PptxParser extends Parser {
     readonly name = "pptx";
     readonly extensions: readonly [".pptx"];
     parse(input: ParseInput): Promise<ParseOutput>;
 }
-export declare class XlsxParser extends Parser {
-    readonly name = "xlsx";
-    readonly extensions: readonly [".xlsx", ".xls"];
-    parse(input: ParseInput): Promise<ParseOutput>;
-}

package/dist/parser/types.d.ts

@@ -8,7 +8,7 @@ export interface ParseOutput {
     readonly metadata?: Readonly<Record<string, unknown>>;
     readonly diagnostics?: readonly ParseDiagnostic[];
 }
-export type ParseDiagnosticCode = "pdf-extract-thin" | "pdf-hybrid-unavailable";
+export type ParseDiagnosticCode = "pdf-extract-thin" | "parser-warning";
 export interface ParseDiagnostic {
     readonly code: ParseDiagnosticCode;
     readonly severity: "warning";

package/dist/process/terminate-tree.d.ts

@@ -16,5 +16,7 @@ export declare const TREE_KILL_GRACE_MS = 500;
  * live parent/child chain and therefore requires `detached: false` there.
  *
  * Returns whether the process group / tree was signalled.
+ * Default SIGKILL: SIGTERM lets a Node descendant close inherited stdio during
+ * shutdown while the pid is still kill(pid,0)-visible (zombie or exiting).
  */
 export declare function terminateProcessTree(child: ChildProcess, signal?: NodeJS.Signals): boolean;

package/dist/retrieval/types.d.ts

@@ -7,7 +7,7 @@ export interface RetrievalResult {
 }
 export interface RetrievalMethodDescriptor {
     name: string;
-    type: "posix" | "vector" | "bm25" | "hybrid" | "visual";
+    type: "posix" | "vector" | "bm25" | "hybrid" | "visual" | "remote";
     description: string;
     status: "active" | "stub";
     capabilities: string[];

package/licenses/embedding-assets.json

@@ -16,7 +16,11 @@
       "modelAssetId": "qwen3-embedding-0.6b",
       "licenseId": "Apache-2.0",
       "noticeFile": "licenses/qwen3-embedding-notice.txt",
-      "noticeReference": "Qwen3 model card and Apache-2.0 license"
+      "noticeReference": "Qwen3 model card and Apache-2.0 license",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
+      "upstreamCardUrl": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
+      "modelCard": "docs/model-cards/qwen3-embedding-0.6b.md"
     },
     {
       "profileId": "embeddinggemma-300m",
@@ -32,7 +36,11 @@
       "modelAssetId": "embeddinggemma-300m",
       "licenseId": "Gemma Terms of Use",
       "noticeFile": "licenses/gemma-notice.txt",
-      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use"
+      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
+      "upstreamCardUrl": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
+      "modelCard": "docs/model-cards/embeddinggemma-300m.md"
     }
   ],
   "assets": [
@@ -45,7 +53,11 @@
       "sha256": "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
       "licenseId": "Apache-2.0",
       "noticeFile": "licenses/qwen3-embedding-notice.txt",
-      "noticeReference": "Qwen3 model card and Apache-2.0 license"
+      "noticeReference": "Qwen3 model card and Apache-2.0 license",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
+      "upstreamCardUrl": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
+      "modelCard": "docs/model-cards/qwen3-embedding-0.6b.md"
     },
     {
       "kind": "model",
@@ -56,7 +68,11 @@
       "sha256": "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
       "licenseId": "Gemma Terms of Use",
       "noticeFile": "licenses/gemma-notice.txt",
-      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use"
+      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
+      "upstreamCardUrl": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
+      "modelCard": "docs/model-cards/embeddinggemma-300m.md"
     },
     {
       "kind": "runtime",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.5.2",
+  "version": "2.6.0",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -49,31 +49,31 @@
     "node": ">=24.0.0"
   },
   "dependencies": {
-    "@earendil-works/pi-agent-core": "^0.85.1",
-    "@earendil-works/pi-ai": "^0.85.1",
-    "@earendil-works/pi-coding-agent": "0.85.1",
-    "@earendil-works/pi-tui": "0.85.1",
-    "@opendataloader/pdf": "^2.5.7",
-    "@rhwp/core": "0.8.6",
+    "@earendil-works/pi-agent-core": "^0.86.1",
+    "@earendil-works/pi-ai": "^0.86.1",
+    "@earendil-works/pi-coding-agent": "0.86.1",
+    "@earendil-works/pi-tui": "0.86.1",
     "chardet": "^2.2.0",
     "fast-xml-parser": "^5.11.1",
     "iconv-lite": "^0.7.3",
     "jszip": "^3.10.1",
+    "kordoc": "^4.15.7",
     "mailparser": "^3.9.20",
+    "pdfjs-dist": "^6.3.289",
     "puppeteer-core": "^25.11.0",
     "slides-grab": "^1.5.1",
     "smol-toml": "1.8.0",
     "tesseract.js": "^7.0.0",
     "typebox": "^1.3.25",
-    "xlsx": "^0.18.5",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.5.13",
+    "@biomejs/biome": "2.5.14",
     "@types/mailparser": "^3.4.6",
     "@types/node": "^26.4.1",
     "@types/ws": "^8.18.1",
     "typescript": "^7.0.2",
-    "vitest": "^5.0.0"
+    "vitest": "^5.0.0",
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
   }
 }

package/skills/autorag-doctor/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: autorag-doctor
-description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding gateway, and every CLI-backed datasource (katok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the gateway will not start, or the user asks to check, diagnose, verify, or repair AutoRAG.
+description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding gateway, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the gateway will not start, or the user asks to check, diagnose, verify, or repair AutoRAG.
 license: MIT
 ---
 
@@ -16,7 +16,7 @@ Always finish with the status table in [Report](#report).
 ## Safety
 
 - Never delete, move, or edit source documents.
-- Never delete a datasource's **native** store (`~/.katok`, `~/.discrawl`,
+- Never delete a datasource's **native** store (`~/Library/Application Support/katok`, `~/.discrawl`,
   `~/.mailcrawl`, `.qmd`, Telegram/WhatsApp/Notion archives). AutoRAG only
   reads them; rebuilding them is the owning CLI's job.
 - Only AutoRAG-owned state under `AUTORAG_HOME` and the workspace `.autorag`
@@ -58,7 +58,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
 |---|---|---|
 | MinSync (local docs) | `minsync status`, `minsync check`, `minsync verify` | `autorag refresh --method minsync` |
 | Jikji (discovery) | `jikji doctor` | `autorag refresh --method jikji` |
-| KakaoTalk | `katok doctor` | `katok sync && katok index` |
+| KakaoTalk | `lazykatok doctor` | `lazykatok sync && lazykatok index` |
 | Discord | `discrawl --json metadata` | `discrawl sync` |
 | Slack | `slacrawl --json doctor` | `slacrawl sync` |
 | WhatsApp | `wacrawl --json doctor` | `wacrawl import` |
@@ -68,6 +68,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
 | Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
 | Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
 | Spotlight (macOS) | `mdutil -s /` | indexed by the OS; no AutoRAG sync |
+| Lark / Feishu | `lark-cli auth status --format json` | no local sync; search is remote |
 
 Rules:
 

package/skills/autorag-lite-search/SKILL.md

@@ -138,13 +138,63 @@ autorag lite report "key findings in the Q3 report" --input report.json --json
 cat report.json | autorag lite report "key findings in the Q3 report" --json
 ```
 
-The input is the `emit_autorag_results` JSON schema: `answer`, numbered
-`results` (with `number`, `title`, `summary`, `confidence` in `[0, 1]`,
-optional `source`), and a `mapping` whose entries map each result number to
-its `method` and `source`, plus optional `evidenceRefs` and `warnings`.
-Result and mapping numbers must be one-to-one positive integers. Evidence
-confidence values must be in `[0, 1]`. Invalid input exits with code 2 and
-`{"ok": false, "error": "..."}`.
+The input is the `emit_autorag_results` JSON schema. Every field below that is
+not marked optional is required — `results[].evidence` and `mapping[].content`
+are the two that are easiest to forget:
+
+- `answer` (string) — the curated answer, citing results as `[1]`, `[2]`.
+- `results` (array) — one entry per numbered knowledge unit, each with
+  `number` (1-based integer), `title` (string), `summary` (string),
+  `evidence` (array of `{"excerpt": "..."}` objects, each with an optional
+  `lineNumber`), and `confidence` (number in `[0, 1]`). A result carries no
+  `source`; source identity lives in `mapping`.
+- `mapping` (array) — exactly one entry per result `number`, each with
+  `number`, `source` (the opaque identity carried from retrieve), `method`,
+  and `content` (the raw snippet kept for feedback tracking). `evidenceRefs`
+  is optional; each ref needs `method` and `source` and may add `excerpt`,
+  `content`, `chunkIndex`, `lineNumber`, `retrievalResultId`, and
+  `confidence` in `[0, 1]`.
+- `warnings` (optional array of strings).
+
+Result and mapping numbers must be one-to-one positive integers, and every
+`confidence` must be in `[0, 1]`. A complete minimal input:
+
+```json
+{
+  "answer": "[1] Refund exceptions need director approval before payout.",
+  "results": [
+    {
+      "number": 1,
+      "title": "Refund exception approval",
+      "summary": "Refund exceptions require director approval before payout.",
+      "evidence": [
+        {
+          "excerpt": "Refund exceptions require director approval.",
+          "lineNumber": 12
+        }
+      ],
+      "confidence": 0.9
+    }
+  ],
+  "mapping": [
+    {
+      "number": 1,
+      "source": "/Users/me/corpus/refund-policy.txt",
+      "method": "minsync",
+      "content": "Refund exceptions require director approval before payout."
+    }
+  ],
+  "warnings": []
+}
+```
+
+Invalid input exits with code 2 and `{"ok": false, "error": "..."}`. The
+rejection names every field that failed, by JSON path, so fix exactly what it
+points at instead of guessing:
+
+```text
+{"ok": false, "error": "Invalid report: expected the emit_autorag_results JSON schema (/results/0: must have required properties evidence; /mapping/0: must have required properties content)"}
+```
 
 `source` values in a report are opaque identifiers carried from retrieval
 output. Report persistence performs no filesystem reads against them; it

package/skills/autorag-lite-setup/SKILL.md

@@ -59,11 +59,11 @@ include `AUTORAG_HOME`, `AUTORAG_CONFIG`, `AUTORAG_SEARCH_PATHS`,
 
 ## Probe and configure datasources (setup wizard)
 
-`autorag lite ui` is still in development — do not recommend it for datasource
+A datasource setup UI is not shipped in this build — do not recommend it for datasource
 setup. Configure datasources directly in trusted config, wizard-style:
 
 1. Probe every datasource for setup feasibility before asking the user
-   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
    `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`) and its
    local store or archive is present. CLI-backed datasources own their own
    archive, index, and authentication, so environment credentials (such as bot
@@ -88,7 +88,7 @@ Datasource skills belong in trusted config and remain default-deny.
 
 Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
 `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
-`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`) or connection
+`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`) or connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `datasourceAccess.allowedTags` and `allowedScopes` narrow trusted access and
@@ -125,7 +125,7 @@ autorag lite refresh --force --json
   different provider.
 - Exact duplicate exclusion during refresh is enabled by default via the
   external `dupey` CLI. Install dupey during setup when it is missing
-  (`command -v dupey || cargo install dupey`) and tell the user the feature is
+  (`command -v dupey || cargo install dupey --locked`) and tell the user the feature is
   available; when installation is impossible, refresh continues without it
   and the user is told duplicate exclusion is off. Set
   `"excludeExactDuplicates": false` to index every copy.

package/skills/autorag-setup/SKILL.md

@@ -138,7 +138,7 @@ Exact duplicate exclusion is enabled by default. AutoRAG invokes the external
 `dupey` CLI before parsed-mirror indexing, keeps the newest filesystem copy for
 each exact canonical-text hash, and excludes older copies from the mirror.
 Install dupey during setup when it is missing (`command -v dupey || cargo
-install dupey`) and tell the user the feature is available; when installation
+install dupey --locked`) and tell the user the feature is available; when installation
 is impossible, refresh continues without this optimization and the user is
 told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
 index every copy.
@@ -180,12 +180,12 @@ datasources automatically:
 autorag setup --format json
 ```
 
-`autorag ui` is still in development — do not recommend it for datasource
+A datasource setup UI is not shipped in this build — do not recommend it for datasource
 setup. Configure datasources directly in trusted config, wizard-style:
 
 1. Probe every datasource for setup feasibility before asking the user
-   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
-   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`) and its
+   anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`, `lark-cli`) and its
    local store or archive is present. CLI-backed datasources own their own
    archive, index, and authentication, so environment credentials (such as bot
    tokens) are never required or checked for them. Non-CLI connectors
@@ -207,8 +207,8 @@ setup. Configure datasources directly in trusted config, wizard-style:
 
 Datasource skills belong in trusted config and remain default-deny. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
-`clawgallery`, `notion`, `github`, `cloud-drive`, `mail-export`,
-`mailcrawl`, `obsidian`, `rss`, and `spotlight`. Config keys may be connection
+`clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
+`mailcrawl`, `obsidian`, `rss`, `spotlight`, and `lark`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `scope` and tags can narrow trusted access but cannot grant it.
@@ -217,6 +217,7 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
 {
   "datasources": {
     "github": { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
+    "github-gist": { "connector": { "tokenEnv": "GITHUB_TOKEN" } },
     "google-drive": { "type": "cloud-drive", "connector": { "provider": "google-drive", "remote": "gdrive:" } },
     "archive-drive": { "type": "cloud-drive", "connector": { "remote": "archive:" } },
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
@@ -233,11 +234,13 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
 Tokens are environment-variable names, not raw secrets. CLI-backed connectors
 keep authentication in their external tool configuration.
 
-Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.1.6` or newer)
+Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.0` or newer)
 and configured through its own Himalaya account. AutoRAG runs its local `sync`
 and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
 hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
-fails with `text array must be non-empty`.
+fails with `text array must be non-empty`. 0.2.0 defaults to the in-process
+native `Qwen/Qwen3-Embedding-0.6B` embedder and keeps vectors in LanceDB, so a
+cold cache makes the first `index` download ONNX weights and run for minutes.
 Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
 datasource is removed.