@autorag/librarian 2.5.2 → 2.5.3

package/dist/process/terminate-tree.d.ts

@@ -16,5 +16,7 @@ export declare const TREE_KILL_GRACE_MS = 500;
  * live parent/child chain and therefore requires `detached: false` there.
  *
  * Returns whether the process group / tree was signalled.
+ * Default SIGKILL: SIGTERM lets a Node descendant close inherited stdio during
+ * shutdown while the pid is still kill(pid,0)-visible (zombie or exiting).
  */
 export declare function terminateProcessTree(child: ChildProcess, signal?: NodeJS.Signals): boolean;

package/dist/retrieval/types.d.ts

@@ -7,7 +7,7 @@ export interface RetrievalResult {
 }
 export interface RetrievalMethodDescriptor {
     name: string;
-    type: "posix" | "vector" | "bm25" | "hybrid" | "visual";
+    type: "posix" | "vector" | "bm25" | "hybrid" | "visual" | "remote";
     description: string;
     status: "active" | "stub";
     capabilities: string[];

package/licenses/embedding-assets.json

@@ -16,7 +16,11 @@
       "modelAssetId": "qwen3-embedding-0.6b",
       "licenseId": "Apache-2.0",
       "noticeFile": "licenses/qwen3-embedding-notice.txt",
-      "noticeReference": "Qwen3 model card and Apache-2.0 license"
+      "noticeReference": "Qwen3 model card and Apache-2.0 license",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
+      "upstreamCardUrl": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
+      "modelCard": "docs/model-cards/qwen3-embedding-0.6b.md"
     },
     {
       "profileId": "embeddinggemma-300m",
@@ -32,7 +36,11 @@
       "modelAssetId": "embeddinggemma-300m",
       "licenseId": "Gemma Terms of Use",
       "noticeFile": "licenses/gemma-notice.txt",
-      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use"
+      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
+      "upstreamCardUrl": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
+      "modelCard": "docs/model-cards/embeddinggemma-300m.md"
     }
   ],
   "assets": [
@@ -45,7 +53,11 @@
       "sha256": "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
       "licenseId": "Apache-2.0",
       "noticeFile": "licenses/qwen3-embedding-notice.txt",
-      "noticeReference": "Qwen3 model card and Apache-2.0 license"
+      "noticeReference": "Qwen3 model card and Apache-2.0 license",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
+      "upstreamCardUrl": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
+      "modelCard": "docs/model-cards/qwen3-embedding-0.6b.md"
     },
     {
       "kind": "model",
@@ -56,7 +68,11 @@
       "sha256": "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
       "licenseId": "Gemma Terms of Use",
       "noticeFile": "licenses/gemma-notice.txt",
-      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use"
+      "noticeReference": "Google EmbeddingGemma model card and Gemma Terms of Use",
+      "intendedUse": "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
+      "unsuitableUse": "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
+      "upstreamCardUrl": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
+      "modelCard": "docs/model-cards/embeddinggemma-300m.md"
     },
     {
       "kind": "runtime",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.5.2",
+  "version": "2.5.3",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {

package/skills/autorag-doctor/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: autorag-doctor
-description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding gateway, and every CLI-backed datasource (katok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the gateway will not start, or the user asks to check, diagnose, verify, or repair AutoRAG.
+description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding gateway, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the gateway will not start, or the user asks to check, diagnose, verify, or repair AutoRAG.
 license: MIT
 ---
 
@@ -16,7 +16,7 @@ Always finish with the status table in [Report](#report).
 ## Safety
 
 - Never delete, move, or edit source documents.
-- Never delete a datasource's **native** store (`~/.katok`, `~/.discrawl`,
+- Never delete a datasource's **native** store (`~/Library/Application Support/katok`, `~/.discrawl`,
   `~/.mailcrawl`, `.qmd`, Telegram/WhatsApp/Notion archives). AutoRAG only
   reads them; rebuilding them is the owning CLI's job.
 - Only AutoRAG-owned state under `AUTORAG_HOME` and the workspace `.autorag`
@@ -58,7 +58,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
 |---|---|---|
 | MinSync (local docs) | `minsync status`, `minsync check`, `minsync verify` | `autorag refresh --method minsync` |
 | Jikji (discovery) | `jikji doctor` | `autorag refresh --method jikji` |
-| KakaoTalk | `katok doctor` | `katok sync && katok index` |
+| KakaoTalk | `lazykatok doctor` | `lazykatok sync && lazykatok index` |
 | Discord | `discrawl --json metadata` | `discrawl sync` |
 | Slack | `slacrawl --json doctor` | `slacrawl sync` |
 | WhatsApp | `wacrawl --json doctor` | `wacrawl import` |
@@ -68,6 +68,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
 | Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
 | Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
 | Spotlight (macOS) | `mdutil -s /` | indexed by the OS; no AutoRAG sync |
+| Lark / Feishu | `lark-cli auth status --format json` | no local sync; search is remote |
 
 Rules:
 

package/skills/autorag-lite-setup/SKILL.md

@@ -63,7 +63,7 @@ include `AUTORAG_HOME`, `AUTORAG_CONFIG`, `AUTORAG_SEARCH_PATHS`,
 setup. Configure datasources directly in trusted config, wizard-style:
 
 1. Probe every datasource for setup feasibility before asking the user
-   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
    `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`) and its
    local store or archive is present. CLI-backed datasources own their own
    archive, index, and authentication, so environment credentials (such as bot
@@ -88,7 +88,7 @@ Datasource skills belong in trusted config and remain default-deny.
 
 Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
 `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
-`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`) or connection
+`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`) or connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `datasourceAccess.allowedTags` and `allowedScopes` narrow trusted access and

package/skills/autorag-setup/SKILL.md

@@ -184,8 +184,8 @@ autorag setup --format json
 setup. Configure datasources directly in trusted config, wizard-style:
 
 1. Probe every datasource for setup feasibility before asking the user
-   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
-   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`) and its
+   anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`, `lark-cli`) and its
    local store or archive is present. CLI-backed datasources own their own
    archive, index, and authentication, so environment credentials (such as bot
    tokens) are never required or checked for them. Non-CLI connectors
@@ -207,8 +207,8 @@ setup. Configure datasources directly in trusted config, wizard-style:
 
 Datasource skills belong in trusted config and remain default-deny. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
-`clawgallery`, `notion`, `github`, `cloud-drive`, `mail-export`,
-`mailcrawl`, `obsidian`, `rss`, and `spotlight`. Config keys may be connection
+`clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
+`mailcrawl`, `obsidian`, `rss`, `spotlight`, and `lark`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `scope` and tags can narrow trusted access but cannot grant it.
@@ -217,6 +217,7 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
 {
   "datasources": {
     "github": { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
+    "github-gist": { "connector": { "tokenEnv": "GITHUB_TOKEN" } },
     "google-drive": { "type": "cloud-drive", "connector": { "provider": "google-drive", "remote": "gdrive:" } },
     "archive-drive": { "type": "cloud-drive", "connector": { "remote": "archive:" } },
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
@@ -233,11 +234,13 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
 Tokens are environment-variable names, not raw secrets. CLI-backed connectors
 keep authentication in their external tool configuration.
 
-Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.1.6` or newer)
+Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.0` or newer)
 and configured through its own Himalaya account. AutoRAG runs its local `sync`
 and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
 hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
-fails with `text array must be non-empty`.
+fails with `text array must be non-empty`. 0.2.0 defaults to the in-process
+native `Qwen/Qwen3-Embedding-0.6B` embedder and keeps vectors in LanceDB, so a
+cold cache makes the first `index` download ONNX weights and run for minutes.
 Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
 datasource is removed.
 

package/dist/agent/search-datasource-tool.d.ts

@@ -1,30 +0,0 @@
-import type { AgentTool } from "@earendil-works/pi-agent-core";
-import { Type } from "typebox";
-import type { RetrievalDiagnostic, RetrievalResult } from "../retrieval/types.js";
-import { type SearchDocumentRetrievalTraceResult } from "./search-documents.js";
-export declare const SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME = "search_datasource_documents";
-declare const searchDatasourceSchema: Type.TObject<{
-    query: Type.TString;
-    topK: Type.TOptional<Type.TInteger>;
-    scope: Type.TOptional<Type.TString>;
-}>;
-export interface SearchDatasourceDocumentsDetails {
-    readonly method: "datasource";
-    readonly resultCount: number;
-    readonly sources: readonly string[];
-    readonly diagnostics: readonly RetrievalDiagnostic[];
-    /** Top candidates in traceable shape (additive; used for the run's retrieval trace). */
-    readonly results?: readonly SearchDocumentRetrievalTraceResult[];
-}
-export interface DatasourceSearchProvider {
-    searchDatasourceDocuments(query: string, options?: {
-        readonly topK?: number;
-        readonly scope?: string;
-    }): Promise<{
-        readonly results: readonly RetrievalResult[];
-        readonly diagnostics: readonly RetrievalDiagnostic[];
-    }>;
-}
-export declare function createSearchDatasourceDocumentsTool(provider: DatasourceSearchProvider): AgentTool<typeof searchDatasourceSchema, SearchDatasourceDocumentsDetails>;
-export declare function formatDatasourceResults(results: readonly RetrievalResult[], diagnostics: readonly RetrievalDiagnostic[]): string;
-export {};

package/dist/datasource/skills/katok/client.d.ts

@@ -1,21 +0,0 @@
-import type { KatokChunkResult, KatokContextResult, KatokDoctorResult, KatokIndexResult, KatokOptions, KatokParentResult, KatokSearchMode, KatokSearchOptions, KatokSearchResult, KatokSyncResult } from "./types.js";
-/**
- * Thin external `katok` CLI wrapper. Every method spawns the `katok` binary as
- * a child process, parses JSON from stdout, and returns a discriminated ok/fail
- * union. No method throws for expected failures (missing binary, CLI nonzero
- * exit, timeout, oversized output, or invalid JSON). The client never opens the
- * KakaoTalk database directly.
- */
-export declare class KatokClient {
-    private readonly options;
-    constructor(options?: KatokOptions);
-    doctor(signal?: AbortSignal): Promise<KatokDoctorResult>;
-    sync(signal?: AbortSignal): Promise<KatokSyncResult>;
-    index(signal?: AbortSignal): Promise<KatokIndexResult>;
-    search(mode: KatokSearchMode, query: string, options?: KatokSearchOptions): Promise<KatokSearchResult>;
-    chunkGet(chunkId: string, signal?: AbortSignal): Promise<KatokChunkResult>;
-    context(chunkId: string, signal?: AbortSignal): Promise<KatokContextResult>;
-    parent(chunkId: string, signal?: AbortSignal): Promise<KatokParentResult>;
-    /** Single retrieval pipeline: build env, spawn, parse-free raw result. */
-    private run;
-}

package/dist/datasource/skills/katok/index.d.ts

@@ -1,12 +0,0 @@
-/**
- * KakaoTalk datasource skill barrel.
- *
- * Re-exports the retrieval methods and skill plus the underlying client/types
- * from sibling modules. Public source paths are path-opaque virtual paths of
- * the form `/kakao/<instance-id>/chunks/<chunk-id>`.
- */
-export { KatokClient } from "./client.js";
-export { KatokBm25Method, type KatokMethodOptions, type KatokSearchClient, KatokSemanticMethod } from "./methods.js";
-export { KatokSkill, type KatokSkillClient, type KatokSkillOptions } from "./skill.js";
-export type { KatokChunkResult, KatokDoctorResult, KatokFailureReason, KatokHit, KatokIndexResult, KatokOptions, KatokSearchMode, KatokSearchOptions, KatokSearchResult, KatokSyncResult, } from "./types.js";
-export { DEFAULT_KATOK_OPTIONS } from "./types.js";

package/dist/datasource/skills/katok/paths.d.ts

@@ -1,40 +0,0 @@
-/**
- * Path helpers for the KakaoTalk (`katok`) datasource skill.
- *
- * All paths are workspace-relative and live under
- * `<workspaceRoot>/.autorag/datasources/katok`. The client never exposes raw
- * filesystem paths to callers; chunk identifiers are surfaced as opaque
- * slash-hierarchical sources of the form
- * `/kakao/<instance-id>/chunks/<chunk-id>`.
- */
-export declare const AUTORAG_DIRNAME = ".autorag";
-export declare const DATASOURCES_DIRNAME = "datasources";
-export declare const KATOK_DATASOURCE_DIRNAME = "katok";
-/** Logical datasource kind for source identifiers. */
-export declare const KATOK_SOURCE_KIND: "kakao";
-/**
- * Root directory for katok-managed state under a workspace. This is where the
- * external `katok` CLI stores its index, chunks, and sync state.
- */
-export declare function katokDatasourceRoot(workspaceRoot: string): string;
-/** Directory holding the katok chunk index. */
-export declare function katokIndexPath(workspaceRoot: string): string;
-/** Directory holding materialized katok chunks. */
-export declare function katokChunksPath(workspaceRoot: string): string;
-/** Directory holding katok sync state. */
-export declare function katokSyncPath(workspaceRoot: string): string;
-/**
- * Builds the path-opaque, slash-hierarchical source identifier for a single
- * katok chunk. Callers (the KatokSkill methods) use this to populate
- * `RetrievalResult.source` so no real filesystem path ever leaks.
- */
-export declare function katokSourcePath(instanceId: string, chunkId: string): string;
-/**
- * Parses a katok source identifier back into its `{ instanceId, chunkId }`
- * components. Returns `undefined` for malformed or non-kakao sources so that
- * deny decisions remain explicit rather than undefined-as-deny.
- */
-export declare function parseKatokSourcePath(source: string): {
-    readonly instanceId: string;
-    readonly chunkId: string;
-} | undefined;

package/dist/datasource/skills/katok/types.d.ts

@@ -1,123 +0,0 @@
-import type { RetrievalOptions } from "../../../retrieval/types.js";
-/**
- * Search modes accepted by the external `katok search` subcommand.
- *
- * - `semantic` — dense vector similarity over indexed chunks.
- * - `keyword`  — exact / tokenized keyword matching.
- * - `hybrid`   — fused semantic + keyword scores.
- */
-export type KatokSearchMode = "semantic" | "keyword" | "hybrid";
-/**
- * Configuration for {@link KatokClient}. All fields optional; sensible defaults
- * mirror the jikji client. The client spawns the `katok` binary as a child
- * process — it never opens the KakaoTalk database directly.
- */
-export interface KatokOptions {
-    /** Explicit path to the `katok` binary. Defaults to a bare `katok` PATH lookup. */
-    readonly binaryPath?: string;
-    /** Spawn timeout in milliseconds. Default 60_000 (sync/index over large archives can be slow). */
-    readonly timeoutMs?: number;
-    /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
-    readonly maxBufferBytes?: number;
-    /** Explicit native katok data directory, passed as `--data-dir`. */
-    readonly workspacePath?: string;
-    /** Explicit operator-owned configuration/workspace transport. */
-    readonly configPath?: string;
-    /** Environment overrides merged on top of `process.env` for the child. */
-    readonly env?: Readonly<Record<string, string | undefined>>;
-}
-export declare const DEFAULT_KATOK_BINARY = "katok";
-export declare const DEFAULT_KATOK_TIMEOUT_MS = 60000;
-export declare const DEFAULT_KATOK_MAX_BUFFER_BYTES = 1048576;
-export declare const DEFAULT_KATOK_OPTIONS: {
-    readonly binaryPath: "katok";
-    readonly timeoutMs: 60000;
-    readonly maxBufferBytes: 1048576;
-};
-/**
- * Reasons a katok CLI invocation can fail. The client never throws for these —
- * every method returns a discriminated union with `ok: false` and one of these
- * reasons.
- */
-export type KatokFailureReason = "binary-missing" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
-/** Common failure payload shared by every method result union. */
-export interface KatokFailure {
-    readonly ok: false;
-    readonly reason: KatokFailureReason;
-    /** Raw stdout captured (may be partial / empty). Never a binary path. */
-    readonly stdout: string;
-    /** Raw stderr captured (may be partial / empty). Never a binary path. */
-    readonly stderr: string;
-    /** Exit code, or `null` when the process never exited normally. */
-    readonly code: number | null;
-    readonly hits?: readonly unknown[];
-}
-/** A single indexed chat chunk returned by the katok CLI. */
-export interface KatokChunk {
-    readonly chunkId: string;
-    readonly content: string;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-/** A scored search hit. */
-export interface KatokSearchHit {
-    readonly chunkId: string;
-    readonly score: number;
-    readonly content: string;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-    readonly source?: string;
-}
-export type KatokHit = KatokSearchHit;
-/** Result of `katok doctor`. */
-export interface KatokDoctorInfo {
-    readonly version?: string;
-    readonly ready: boolean;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-/** Result of `katok sync`. */
-export interface KatokSyncInfo {
-    readonly synced: boolean;
-    readonly messageCount?: number;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-/** Result of `katok index`. */
-export interface KatokIndexInfo {
-    readonly chunkCount: number;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-/** Context payload: the chunks surrounding a target chunk. */
-export interface KatokContext {
-    readonly chunks: readonly KatokChunk[];
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-/** Discriminated ok-success shape carrying typed `data`. */
-export interface KatokOk<T> {
-    readonly ok: true;
-    readonly data: T;
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number;
-}
-export interface KatokSearchOk {
-    readonly ok: true;
-    readonly hits: readonly KatokSearchHit[];
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number;
-    readonly data: {
-        readonly hits: readonly KatokSearchHit[];
-    };
-}
-export type KatokDoctorResult = KatokOk<KatokDoctorInfo> | KatokFailure;
-export type KatokSyncResult = KatokOk<KatokSyncInfo> | KatokFailure;
-export type KatokIndexResult = KatokOk<KatokIndexInfo> | KatokFailure;
-export type KatokSearchResult = KatokSearchOk | KatokFailure;
-export type KatokChunkResult = KatokOk<KatokChunk> | KatokFailure;
-export type KatokContextResult = KatokOk<KatokContext> | KatokFailure;
-export type KatokParentResult = KatokOk<KatokChunk> | KatokFailure;
-/**
- * Search-specific options. Reuses the shared {@link RetrievalOptions} so the
- * KatokSkill retrieval method can pass its options straight through. The
- * client maps `topK` to the native CLI limit flag; other fields are handled
- * by the retrieval layer or are not supported by katok.
- */
-export type KatokSearchOptions = RetrievalOptions;