@autorag/librarian 2.10.1 → 2.10.3

package/dist/parser/types.d.ts

@@ -17,5 +17,10 @@ export interface ParseDiagnostic {
 export declare abstract class Parser {
     abstract readonly name: string;
     abstract readonly extensions: readonly string[];
+    /**
+     * Stamp recorded next to a per-file skip. Refresh retries a source a parser failed on exactly
+     * once when this changes, so bump it whenever parse behavior changes (including the engine behind it).
+     */
+    readonly version: string;
     abstract parse(input: ParseInput): Promise<ParseOutput>;
 }

package/dist/process/abort-reason.d.ts

@@ -0,0 +1,2 @@
+/** The rejection for work its caller aborted: the signal's reason when it is an error, else an `AbortError`. */
+export declare function abortReason(signal: AbortSignal): Error;

package/dist/retrieval/index.d.ts

@@ -3,7 +3,7 @@ export { RetrievalEngine, type RetrievalEngineOptions } from "./engine.js";
 export type { MergeOptions } from "./merger.js";
 export { ParallelRetriever, ResultMerger } from "./merger.js";
 export { RetrievalMethodRegistry } from "./registry.js";
-export { type CreateRerankerOptions, createReranker, DEFAULT_AUTORAG_RERANK_API_KEY_ENV, DEFAULT_AUTORAG_RERANK_MODEL, DEFAULT_RERANK_API_KEY_ENV, DEFAULT_RERANK_MODEL, DEFAULT_RERANK_PROVIDER, DEFAULT_RERANK_TOP_N, defaultRerankApiKeyEnv, defaultRerankModel, OpenRouterReranker, type OpenRouterRerankerOptions, type Reranker, type RerankerDescriptor, type RerankOptions, SUPPORTED_RERANK_PROVIDERS, } from "./rerank.js";
+export { type CreateRerankerOptions, createReranker, DEFAULT_DAZZI_RERANK_MODEL, DEFAULT_RERANK_API_KEY_ENV, DEFAULT_RERANK_MODEL, DEFAULT_RERANK_PROVIDER, DEFAULT_RERANK_TOP_N, defaultRerankApiKeyEnv, defaultRerankModel, OpenRouterReranker, type OpenRouterRerankerOptions, type Reranker, type RerankerDescriptor, type RerankOptions, SUPPORTED_RERANK_PROVIDERS, } from "./rerank.js";
 export { matchesVirtualPathScope, normalizeVirtualPath, normalizeVirtualPathScope, virtualPathScopeToRegExp, } from "./scope.js";
 export { type DatasourceCatalogEntry, derivedDatasourceIds, type RetrievalSelection, RetrievalSelectionError, type RetrievalSelectionErrorCode, resolveSelectedMethods, } from "./selection.js";
 export { describeRetrievalError, groupUnsearchedSurfaces, MINSYNC_SURFACE, type RetrievalSkip, retrievalSurfaceFor, } from "./skip.js";

package/dist/retrieval/rerank.d.ts

@@ -19,21 +19,22 @@ export declare const DEFAULT_RERANK_PROVIDER = "openrouter";
 /** Default environment variable holding the OpenRouter API key. */
 export declare const DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY";
 /**
- * Default rerank model for the hosted AutoRAG (Dazzi) provider — the id the
+ * Default rerank model for the hosted Dazzi Models provider — the id the
  * server lists under `GET <root>/v1/models?operation=rerank`. Rerankers are not
  * chat LLMs, so the project's 2026+ model rule does not cover them.
  */
-export declare const DEFAULT_AUTORAG_RERANK_MODEL = "qwen/qwen3-reranker-8b";
-/** Environment variable holding the AutoRAG account key for the `autorag` rerank provider. */
-export declare const DEFAULT_AUTORAG_RERANK_API_KEY_ENV = "AUTORAG_API_KEY";
+export declare const DEFAULT_DAZZI_RERANK_MODEL = "qwen/qwen3-reranker-8b";
 /** Default number of merged chunks kept after reranking. */
 export declare const DEFAULT_RERANK_TOP_N = 25;
 /** Rerank providers {@link createReranker} can build. */
 export declare const SUPPORTED_RERANK_PROVIDERS: readonly string[];
 /** Default wire model for a rerank provider. */
 export declare function defaultRerankModel(provider: string): string;
-/** Default API-key environment variable for a rerank provider. */
-export declare function defaultRerankApiKeyEnv(provider: string): string;
+/**
+ * Default API-key environment variable for a rerank provider, or `undefined`
+ * for Dazzi Models, which authenticates with the stored `/login` credential.
+ */
+export declare function defaultRerankApiKeyEnv(provider: string): string | undefined;
 /** Static description of a reranker, used for diagnostics and inspection. */
 export interface RerankerDescriptor {
     /** Stable label (e.g. `openrouter`). */
@@ -92,10 +93,15 @@ export interface OpenRouterRerankerOptions {
     readonly provider?: string;
     /** Wire model id. @default DEFAULT_RERANK_MODEL */
     readonly model?: string;
-    /** API key. When omitted, read from `apiKeyEnv` in the environment. */
+    /**
+     * API key. When omitted, `openrouter` reads `apiKeyEnv` from the environment
+     * and `dazzi` uses the stored `/login` credential.
+     */
     readonly apiKey?: string;
-    /** Environment variable holding the API key. @default DEFAULT_RERANK_API_KEY_ENV */
+    /** Environment variable holding the API key (`openrouter` only). @default DEFAULT_RERANK_API_KEY_ENV */
     readonly apiKeyEnv?: string;
+    /** Pi agent home whose `auth.json` holds the Dazzi `/login` credential (`dazzi` only). Default: Pi's agent dir. */
+    readonly agentDir?: string;
     /** Override the OpenRouter base URL (e.g. a self-hosted or gateway endpoint). */
     readonly baseUrl?: string;
     /** Per-request timeout in milliseconds. */
@@ -119,9 +125,9 @@ export declare function createReranker(config: CreateRerankerOptions | false | u
 /**
  * Reranker backed by an OpenRouter-compatible rerank router.
  *
- * OpenRouter is the default provider; the hosted AutoRAG (Dazzi) plan serves the
+ * OpenRouter is the default provider; the hosted Dazzi Models plan serves the
  * same wire shape at `<root>/v1/rerank`, so {@link createReranker} reuses this
- * client with that base URL and key env. Documents are sent as their `content`;
+ * client with that base URL and the stored login token. Documents are sent as their `content`;
  * the original `RetrievalResult` for each returned index is preserved and
  * re-scored with the provider's relevance score.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.10.1",
+  "version": "2.10.3",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {
@@ -36,7 +36,7 @@
     "access": "public"
   },
   "scripts": {
-    "build": "bun build src/index.ts src/core.ts src/cli/index.ts src/mcp/index.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
+    "build": "bun build src/index.ts src/core.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
     "prepublishOnly": "bun run typecheck && bun run build",
     "prepare": "bun run build",
     "test": "node scripts/run-test-files.mjs",
@@ -69,8 +69,7 @@
     "slides-grab": "^1.5.2",
     "smol-toml": "1.9.1",
     "tesseract.js": "^7.0.0",
-    "typebox": "^1.3.37",
-    "yaml": "^2.9.1"
+    "typebox": "^1.3.37"
   },
   "devDependencies": {
     "@biomejs/biome": "2.5.15",
@@ -80,6 +79,7 @@
     "@types/ws": "^8.18.2",
     "typescript": "^7.0.2",
     "vitest": "^5.0.4",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
+    "yaml": "^2.9.1"
   }
 }

package/skills/autorag/SKILL.md

@@ -16,7 +16,8 @@ sources, judges evidence, and curates the final answer. There is no subagent or
 separate model role.
 
 AutoRAG reads source documents and writes indexes only under the configured
-workspace `.autorag/` directory and Jikji's per-source `.jikji/` caches. Never
+workspace `.autorag/` directory and Jikji's central store (`<OS data dir>/jikji/`,
+plus a `.jikji_agent_map.md` guide in each indexed root). Never
 move, rename, edit, or delete source files.
 
 ## Preflight

package/skills/autorag-doctor/SKILL.md

@@ -192,6 +192,11 @@ Use `--force` only when incremental refresh does not clear it. For continuous
 freshness install an hourly `autorag watch --once` job (cron, launchd, systemd
 timer, or Task Scheduler).
 
+A file the parser failed on (corrupt PDF, encrypted HWP) or that was unreadable is not
+re-parsed on every refresh: it is skipped until its mtime, size or parser version changes.
+After fixing the cause (permissions, OCR setup), run `autorag refresh --retry-failed --json`
+to retry those files once without re-indexing healthy ones.
+
 ### Missing or wrong setup
 
 - `unknown-datasource-skill`: the config key is not a builtin template and has
@@ -216,6 +221,8 @@ timer, or Task Scheduler).
 | `stale-index` | Sources changed since last refresh | `autorag refresh --method parsed,minsync` |
 | `index-not-ready` | Index missing or never built | `autorag refresh --json` |
 | `minsync-sync-failed` | MinSync indexing failed; the message carries MinSync's own reason | Fix the reported cause, then `autorag refresh --method minsync` |
+| `minsync-file-skipped` | MinSync skipped one file (unreadable, unparsable, or rejected by the embedder) and kept its older chunks; retried on the next sync | Fix that file's permissions/content, then `autorag refresh --method minsync` |
+| `minsync-lock-held` | Another `minsync` run holds `.minsync/lock`, often an orphan left by a killed host; retryable | Wait for it to finish or stop that `minsync` process, then `autorag refresh --method minsync` |
 | `jikji-unavailable` | Jikji binary missing or install failed | Check cargo, retry refresh |
 | `everything-index-failed` | Windows Everything instance could not start or index; message carries ES exit code and stderr | Fix the reported cause, `autorag refresh --method everything --json` |
 | `embedding-identity-mismatch` | Indexed vectors use a different embedder | `autorag index rebuild --method minsync` |

package/skills/autorag-lite-setup/SKILL.md

@@ -19,7 +19,7 @@ Use `autorag-setup` instead when the model-backed librarian must be configured.
   environment-variable names such as `tokenEnv` or `apiKeyEnv`.
 - Never move, rename, edit, or delete source documents. Lite commands write
   indexes only under the configured workspace `.autorag/` directory and
-  Jikji's per-source `.jikji/` caches.
+  Jikji's central store (`<OS data dir>/jikji/`).
 - Do not index system trees, app bundles, caches, credential stores,
   `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
   `.jikji`.
@@ -169,7 +169,7 @@ AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite refresh --jso
   are missing or broken, run a full refresh or return to setup rather than
   silently degrading to lexical-only search.
 - MinSync's default embedder is in-process native Qwen3 embeddings
-  (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.7): no
+  (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.8): no
   embedder flags, no API key, and no external daemon (such as Ollama) are
   needed, and no corpus text leaves the machine. For gateway-profiled
   embeddings, prefetch with `autorag models prefetch --profile qwen3-embedding-0.6b`.

package/skills/autorag-setup/SKILL.md

@@ -164,15 +164,21 @@ GitHub release into `<workspace>/.autorag/bin` (`minSync.autoInstall` defaults
 to true), and Jikji installs `jikji-cli` through cargo (`jikji.autoInstall`
 defaults to true; requires the Rust toolchain). Set `"autoInstall": false` only
 when managing the binary yourself. Refresh is incremental: MinSync syncs only
-changed parsed mirrors, and `jikji prepare` reuses unchanged documents. Roots
-prepare in parallel.
-
-Jikji stores its prepared corpus metadata in a hidden `.jikji` directory
-inside each indexed source root — that is Jikji's native index layout and
-intended behavior, not a misplaced artifact. Tell the user before the first
-`refresh` that `<root>/.jikji` will be created inside every approved document
-root (one per root, alongside the documents), and never delete or edit its
-contents; removing it only forces a full Jikji re-prepare on the next refresh.
+changed parsed mirrors, and `jikji prepare` reuses unchanged documents. Jikji
+roots prepare one at a time (they share one central SQLite file that rejects
+concurrent writers). A search root nested inside, or duplicating, another root
+is skipped for Jikji because the outer root's index already covers it.
+
+Jikji keeps its prepared corpus (search index, parsed text, wiki, routes) in one
+central store, not inside your documents: `<OS data dir>/jikji/index.sqlite`
+plus `<OS data dir>/jikji/roots/<id>/` (macOS `~/Library/Application Support`,
+Linux `$XDG_DATA_HOME` or `~/.local/share`, Windows `%LOCALAPPDATA%`; override
+with the `JIKJI_DATA_DIR` environment variable). The only file `refresh` adds
+inside an approved document root is the `.jikji_agent_map.md` guide. Tell the
+user that before the first `refresh`, and never delete or edit Jikji's central
+store by hand; removing it only forces a full Jikji re-prepare on the next
+refresh. Older Jikji versions wrote a `<root>/.jikji` directory instead; such a
+leftover is now unused and safe for the user to delete.
 
 Exact duplicate exclusion is enabled by default. AutoRAG invokes the external
 `dupey` CLI before parsed-mirror indexing, keeps the newest filesystem copy for
@@ -184,7 +190,7 @@ told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
 index every copy.
 
 MinSync's default embedder is in-process native Qwen3 embeddings
-(`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.7). The
+(`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.8). The
 default needs no embedder flags, no API key, and no external daemon (such as
 Ollama) — all embeddings run in-process locally and privately. For workspaces
 using the loopback llama-server gateway, prefetch the verified model with
@@ -216,8 +222,9 @@ size must be positive integers, and the dimension must match the embedder
 Leave both enabled. They are the recommended setup: they make simple questions
 fast and multi-part questions thorough.
 
-- **Jev** (`jev`, default `{ "backend": "openrouter" }`, model
-  `typesafe/jev-1.13`) runs before the fast answer. It routes each question to
+- **Jev** (`jev`, default `{ "backend": "dazzi" }` when signed in to Dazzi
+  Models, else `{ "backend": "openrouter" }`; model `jev-1.13` on Dazzi,
+  `typesafe/jev-1.13` on OpenRouter) runs before the fast answer. It routes each question to
   local search, web search, a direct answer (general knowledge or small
   talk skips retrieval entirely), or the `config` branch, and decides whether
   to decompose it. On
@@ -232,14 +239,16 @@ fast and multi-part questions thorough.
   decides whether verification is needed, so a complete, evidence-backed fast
   answer ends the run.
 - **Question decomposition** (`queryDecomposition`, default model
+  `dazzi/openai/gpt-6-luna` when signed in to Dazzi Models, else
   `openrouter/qwen/qwen3.7-flash`) splits a multi-part question into at most
   five search queries that run in parallel.
 
-Both use the user's `OPENROUTER_API_KEY`; confirm it is set (`test -n
-"$OPENROUTER_API_KEY"`, never print it) and tell the user Jev routing is on.
-Without the key, routing falls back to a single local search and the run
-always verifies, so searches still work. A `query-route-fallback` diagnostic
-(`autorag search --debug`) shows that state.
+Both need a credential for the chosen backend: a Dazzi Models `/login` (check with
+`autorag models list --provider dazzi --available`), or the user's
+`OPENROUTER_API_KEY` (`test -n "$OPENROUTER_API_KEY"`, never print it). Tell the
+user Jev routing is on. Without a credential, routing falls back to a single
+local search and the run always verifies, so searches still work. A
+`query-route-fallback` diagnostic (`autorag search --debug`) shows that state.
 
 ```json
 {
@@ -309,6 +318,22 @@ and are never settable from model/tool arguments: `maxDocuments`,
 `bandwidthLimit`, and `dryRun` (cloud-drive/rclone). `maxContentChars` defaults
 to 20000 for RSS and 100000 for Spotlight and cloud-drive.
 
+### Refresh safety limits
+
+`refresh` guards the parsed-mirror refresh against deleting a corpus because a
+search root became unavailable. Every field is optional; unknown keys fail
+config resolution.
+
+| Field | Default | Controls |
+|---|---|---|
+| `maxDeleteRatio` | 0.3 | Largest share (0–1) of indexed documents one refresh may delete; a bigger deletion above 100 documents is suppressed (`mass-delete-suppressed`) unless the refresh is forced |
+| `maxScannedEntries` | 2000000 | Directory entries examined per search root before `walk-truncated` |
+| `maxDepth` | 64 | Deepest directory level walked below a root before `walk-truncated` |
+
+A root that is missing, unlistable, or empty while it previously held documents
+keeps its mirrors and reports `search-root-unavailable`; see
+`docs/minsync-setup.md`.
+
 ## Probe and configure datasource skills (setup wizard)
 
 Use `autorag setup` to probe the local runtime, model profile, and known

package/dist/manifest/index.d.ts

@@ -1,2 +0,0 @@
-export { loadManifest, loadManifests } from "./loader.js";
-export type { StoreManifest } from "./types.js";

package/dist/manifest/loader.d.ts

@@ -1,3 +0,0 @@
-import type { StoreManifest } from "./types.js";
-export declare function loadManifest(filePath: string): StoreManifest;
-export declare function loadManifests(dirPath: string): StoreManifest[];

package/dist/manifest/types.d.ts

@@ -1,8 +0,0 @@
-export interface StoreManifest {
-    name: string;
-    description: string;
-    type: "vector" | "bm25" | "hybrid" | "visual";
-    dataDescription: string;
-    contentTypes: string[];
-    config: Record<string, unknown>;
-}