@autorag/librarian 2.13.4 → 2.13.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/NOTICE +3 -0
  2. package/README.md +50 -18
  3. package/dist/agent/agent.d.ts +110 -39
  4. package/dist/agent/cloud-release-extension.d.ts +43 -0
  5. package/dist/agent/decision-backends.d.ts +49 -0
  6. package/dist/agent/decision-model-check.d.ts +92 -0
  7. package/dist/agent/dupey-tool.d.ts +2 -0
  8. package/dist/agent/everything-search-tool.d.ts +1 -0
  9. package/dist/agent/fsearch-search-tool.d.ts +1 -0
  10. package/dist/agent/index.d.ts +4 -1
  11. package/dist/agent/jev-extension.d.ts +27 -32
  12. package/dist/agent/jikji-find-tool.d.ts +5 -0
  13. package/dist/agent/name-candidates.d.ts +88 -0
  14. package/dist/agent/permission-gate.d.ts +2 -5
  15. package/dist/agent/query-routing.d.ts +1 -1
  16. package/dist/agent/search-documents.d.ts +1 -1
  17. package/dist/agent/sllm-decision.d.ts +34 -0
  18. package/dist/cli/commands/duplicates.d.ts +3 -2
  19. package/dist/cli/commands/models.d.ts +2 -0
  20. package/dist/cli/commands/search.d.ts +2 -0
  21. package/dist/cli/config.d.ts +49 -30
  22. package/dist/cli/index.js +13123 -9835
  23. package/dist/cli/setup.d.ts +27 -1
  24. package/dist/cloud/jev-backend.d.ts +26 -4
  25. package/dist/core.js +5772 -2598
  26. package/dist/datasource/index.d.ts +2 -2
  27. package/dist/datasource/skills/obsidian/index.d.ts +1 -1
  28. package/dist/datasource/skills/obsidian/methods.d.ts +1 -1
  29. package/dist/datasource/skills/obsidian/skill.d.ts +1 -0
  30. package/dist/datasource/skills/obsidian/vault-index.d.ts +1 -1
  31. package/dist/datasource/skills/wombat/client.d.ts +17 -0
  32. package/dist/datasource/skills/wombat/index.d.ts +4 -0
  33. package/dist/datasource/skills/wombat/methods.d.ts +15 -0
  34. package/dist/datasource/skills/wombat/skill.d.ts +38 -0
  35. package/dist/datasource/skills/wombat/types.d.ts +89 -0
  36. package/dist/dupey/cli.d.ts +21 -1
  37. package/dist/dupey/filter.d.ts +8 -1
  38. package/dist/dupey/index.d.ts +1 -0
  39. package/dist/dupey/scan-plan.d.ts +23 -0
  40. package/dist/embedding/index.d.ts +3 -1
  41. package/dist/everything/client.d.ts +1 -2
  42. package/dist/filesystem/cloud-placeholder.d.ts +313 -21
  43. package/dist/filesystem/shared-exclusive-queue.d.ts +16 -0
  44. package/dist/fsearch/client.d.ts +3 -2
  45. package/dist/index.d.ts +1 -1
  46. package/dist/index.js +5877 -2663
  47. package/dist/jikji/answer-pack.d.ts +7 -1
  48. package/dist/jikji/client.d.ts +28 -14
  49. package/dist/jikji/diagnostics.d.ts +16 -2
  50. package/dist/jikji/index.d.ts +3 -3
  51. package/dist/jikji/installer.d.ts +11 -5
  52. package/dist/jikji/retry-command.d.ts +10 -0
  53. package/dist/jikji/types.d.ts +47 -1
  54. package/dist/jikji/version.d.ts +20 -0
  55. package/dist/mcp/index.js +5847 -2660
  56. package/dist/mcp/server.d.ts +3 -0
  57. package/dist/minsync/binary.d.ts +10 -5
  58. package/dist/minsync/client.d.ts +7 -10
  59. package/dist/minsync/embedder-config.d.ts +11 -10
  60. package/dist/minsync/errors.d.ts +3 -0
  61. package/dist/minsync/index.d.ts +1 -1
  62. package/dist/minsync/installer.d.ts +1 -1
  63. package/dist/minsync/method.d.ts +1 -0
  64. package/dist/minsync/types.d.ts +0 -9
  65. package/dist/minsync/version.d.ts +20 -0
  66. package/dist/mirror/cloud-fingerprints.d.ts +30 -0
  67. package/dist/mirror/cloud.d.ts +79 -0
  68. package/dist/mirror/index.d.ts +4 -0
  69. package/dist/mirror/sync.d.ts +33 -1
  70. package/dist/parser/csv.d.ts +26 -0
  71. package/dist/parser/defaults.d.ts +1 -1
  72. package/dist/parser/index.d.ts +2 -0
  73. package/dist/parser/kordoc.d.ts +6 -3
  74. package/dist/parser/parse-worker.js +668 -28
  75. package/dist/parser/spreadsheet-render.d.ts +53 -0
  76. package/dist/parser/spreadsheet.d.ts +27 -0
  77. package/dist/profiles/loop-guard.d.ts +8 -2
  78. package/dist/profiles/retrieval-tools.d.ts +2 -1
  79. package/dist/profiles.js +16381 -13206
  80. package/dist/retrieval/skip.d.ts +1 -1
  81. package/dist/retrieval/types.d.ts +1 -1
  82. package/package.json +3 -3
  83. package/skills/autorag/SKILL.md +7 -4
  84. package/skills/autorag-doctor/SKILL.md +8 -3
  85. package/skills/autorag-lite-setup/SKILL.md +4 -4
  86. package/skills/autorag-setup/SKILL.md +53 -30
  87. package/dist/datasource/skills/cloud-drive/index.d.ts +0 -2
  88. package/dist/datasource/skills/cloud-drive/rclone-connector.d.ts +0 -49
  89. package/dist/datasource/skills/cloud-drive/skill.d.ts +0 -14
  90. package/dist/minsync/kiwi.d.ts +0 -17
  91. package/dist/minsync/lexical-language.d.ts +0 -18
@@ -11,7 +11,7 @@
11
11
  * stderr, paths and all.
12
12
  */
13
13
  import type { RetrievalMethodDescriptor, RetrievalUnsearchedSurface } from "./types.js";
14
- /** Surface label shared by every MinSync-backed local method (vector, BM25, hybrid). */
14
+ /** Surface label shared by every MinSync-backed local method (vector, sparse, hybrid). */
15
15
  export declare const MINSYNC_SURFACE = "minsync";
16
16
  /** One method that did not run, with the error that stopped it. */
17
17
  export interface RetrievalSkip {
@@ -7,7 +7,7 @@ export interface RetrievalResult {
7
7
  }
8
8
  export interface RetrievalMethodDescriptor {
9
9
  name: string;
10
- type: "posix" | "vector" | "bm25" | "hybrid" | "visual" | "remote";
10
+ type: "posix" | "vector" | "bm25" | "sparse" | "hybrid" | "visual" | "remote";
11
11
  description: string;
12
12
  status: "active" | "stub";
13
13
  capabilities: string[];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@autorag/librarian",
3
- "version": "2.13.4",
3
+ "version": "2.13.5",
4
4
  "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -78,7 +78,8 @@
78
78
  "slides-grab": "^1.5.2",
79
79
  "smol-toml": "1.9.1",
80
80
  "tesseract.js": "^7.0.0",
81
- "typebox": "^1.3.37"
81
+ "typebox": "^1.3.37",
82
+ "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
82
83
  },
83
84
  "devDependencies": {
84
85
  "@biomejs/biome": "2.6.4",
@@ -87,7 +88,6 @@
87
88
  "@types/ws": "^8.18.2",
88
89
  "typescript": "^7.0.2",
89
90
  "vitest": "^5.0.5",
90
- "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
91
91
  "yaml": "^2.9.1"
92
92
  }
93
93
  }
@@ -81,8 +81,9 @@ A folder entry excludes everything under it. Removing an entry and running
81
81
  `autorag refresh --method minsync` indexes the file again. Excluded sources are
82
82
  recorded as `user-excluded` skips, so they are not reported as stale. Jikji
83
83
  indexes the source folders directly and is not refreshed here, so AutoRAG also
84
- drops excluded paths from the `jikji_find` answer pack and the baseline prefetch
85
- at retrieval time; the on-disk `.jikji_agent_map.md` stays complete, and direct
84
+ drops excluded paths from the `jikji_find` answer pack and from the files Jikji
85
+ hands to the verification step as starting points, at retrieval time; the
86
+ on-disk `.jikji_agent_map.md` stays complete, and direct
86
87
  file reads remain available.
87
88
 
88
89
  `status` is model-free and path-opaque. `health` resolves the single model,
@@ -223,8 +224,10 @@ grant an agent more than its profile has. `retrieval` optionally adds
223
224
  Retrieval memory is never connected to an automation agent.
224
225
 
225
226
  Every agent run gets a loop guard appended to its system prompt, naming the
226
- rule's folders, file types and events, and forbidding any change that would
227
- fire the rule again, editing the rules file, and running `autorag automate`. At
227
+ rule's folders, file types and events, and how the rule decides what fires it.
228
+ The agent must make no change that fires the rule again (a rename or write whose
229
+ events the rule does not pick up is fine), designs a non-firing change before it
230
+ declines, and never edits the rules file or runs `autorag automate`. At
228
231
  most one agent run is in flight across all rules, and an agent rule is
229
232
  disabled after 6 runs within 60 seconds (command rules: 20). The run's final
230
233
  reply is the `stdout` of its log entry.
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: autorag-doctor
3
- description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl, rclone), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
3
+ description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
4
4
  license: Apache-2.0
5
5
  ---
6
6
 
@@ -72,7 +72,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
72
72
  | Notion | `notcrawl doctor`, `notcrawl status` | `notcrawl sync --source desktop` |
73
73
  | Obsidian / notes | `autorag status` (AutoRAG-owned index) | `autorag refresh --method datasources`; if corrupt, delete `.autorag/datasources/obsidian/<instance>` first |
74
74
  | Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
75
- | Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
75
+ | Coding-agent sessions | `wombat doctor --json` | `wombat sync --index` |
76
76
  | Lark / Feishu | `lark-cli auth status --format json` | no local sync; search is remote |
77
77
 
78
78
  Rules:
@@ -229,6 +229,10 @@ to retry those files once without re-indexing healthy ones.
229
229
  (`cargo install minsync`) or leave `minSync.autoInstall` on, then retry.
230
230
  - `jikji-unavailable`: the binary is missing and auto-install failed. It
231
231
  installs through cargo; verify the Rust toolchain, then `autorag refresh --method jikji` to retry.
232
+ - `jikji-unsupported-version`: the `jikji` AutoRAG found is older than 0.2.0, so it was never run.
233
+ The message names the detected version and the binary path. `autorag refresh --method jikji`
234
+ installs the pinned build; with `jikji.autoInstall: false` or `jikji.binaryPath` upgrade it yourself
235
+ (`cargo install jikji-cli --version 0.2.1 --locked --force`).
232
236
  - `datasource-auth-error` / `datasource-rate-limited`: datasource credentials or quota. Report the
233
237
  missing environment-variable **name** and let the user supply it.
234
238
  - A configured datasource that returns nothing is a native store, connector, or
@@ -244,8 +248,8 @@ to retry those files once without re-indexing healthy ones.
244
248
  | `minsync-sync-failed` | MinSync indexing failed; the message carries MinSync's own reason | Fix the reported cause, then `autorag refresh --method minsync` |
245
249
  | `minsync-file-skipped` | MinSync skipped one file (unreadable, unparsable, or rejected by the embedder) and kept its older chunks; retried on the next sync | Fix that file's permissions/content, then `autorag refresh --method minsync` |
246
250
  | `minsync-lock-held` | Another `minsync` run holds `.minsync/lock`, often an orphan left by a killed host; retryable | Wait for it to finish or stop that `minsync` process, then `autorag refresh --method minsync` |
247
- | `lexical-tokenizer-unavailable` | Warning: the BM25 tokenizer needs the Kiwi library and model, which were not found, so MinSync tokenizes on whitespace and `인증서를` does not match `인증서` | Install Kiwi (set `KIWI_LIBRARY_PATH` and `KIWI_MODEL_PATH`, or install under `~/.local/kiwi`), then `autorag refresh --method minsync` |
248
251
  | `jikji-unavailable` | Jikji binary missing or install failed | Check cargo, retry refresh |
252
+ | `jikji-unsupported-version` | Jikji older than 0.2.0 was found and not run; message names the version and path | `autorag refresh --method jikji` installs the pinned build, or upgrade it yourself |
249
253
  | `everything-index-failed` | Windows Everything instance could not start or index; message carries ES exit code and stderr | Fix the reported cause, `autorag refresh --method everything --json` |
250
254
  | `embedding-identity-mismatch` | Indexed vectors use a different embedder | `autorag index rebuild --method minsync` |
251
255
  | `embedder-unavailable` | No remote endpoint answered, no daemon could start, and the in-process addon is missing or failed | Run `minsync serve` by hand to read its error, install MinSync (`cargo install minsync`) or `@nomadamas/minsync-native`, or configure a remote `minSync.embedder` |
@@ -253,6 +257,7 @@ to retry those files once without re-indexing healthy ones.
253
257
  | `datasource-empty` | Store has no matching content | Re-sync with the CLI, or accept as empty |
254
258
  | `unknown-datasource-skill` | Config name is not a known template | Fix the name or add `"type"` |
255
259
  | `retrieval-method-failed` | One method errored during the query | Read `--debug` diagnostics |
260
+ | `rerank-disabled` | Warning: the `rerank` block names no model, so reranking is off and each source contributes its own top 10 to the first answer | Set `rerank.model` to turn it on, or remove the block; nothing else is wrong |
256
261
  | `datasource-auth-error` | Datasource credentials missing or rejected | Report the env var name |
257
262
  | `query-route-fallback` | Jev routing unavailable (often `OPENROUTER_API_KEY` unset); searched local with the original question | `test -n "$OPENROUTER_API_KEY"`; report the env var name, never its value |
258
263
  | `query-decomposition-failed` | Decomposition model call failed; searched the original question | Check `queryDecomposition.model` resolves (`autorag models list --provider openrouter`) |
@@ -56,7 +56,7 @@ explicit path (`--config` or `AUTORAG_CONFIG`); `--force` refuses to replace the
56
56
  implicit `~/.autorag/config.json`. Explicit user paths win;
57
57
  otherwise propose one to three document-dense roots and get approval before
58
58
  indexing. Supported parsed formats are `md`, `markdown`, `txt`, `text`, `pdf`,
59
- `docx`, `pptx`, `xlsx`, `xls`, `hwp`, `hwpx`, and `eml`. Legacy `.doc` is not
59
+ `docx`, `pptx`, `xlsx`, `xlsm`, `xlsb`, `xls`, `ods`, `csv`, `tsv`, `hwp`, `hwpx`, and `eml`. Legacy `.doc` is not
60
60
  a supported parsed format.
61
61
 
62
62
  Config resolution follows the usual order: `--config`, `AUTORAG_CONFIG`,
@@ -113,7 +113,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
113
113
 
114
114
  1. Probe every datasource for setup feasibility before asking the user
115
115
  anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
116
- `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`) and its
116
+ `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `wombat`) and its
117
117
  local store or archive is present. CLI-backed datasources own their own
118
118
  archive, index, and authentication, so environment credentials (such as bot
119
119
  tokens) are never required or checked for them. Non-CLI connectors
@@ -136,8 +136,8 @@ setup. Configure datasources directly in trusted config, wizard-style:
136
136
  Datasource skills belong in trusted config.
137
137
 
138
138
  Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
139
- `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
140
- `mailcrawl`, `obsidian`, `rss`, `lark`, `hister`) or connection
139
+ `slack`, `discord`, `clawgallery`, `notion`, `github`,
140
+ `mailcrawl`, `obsidian`, `rss`, `lark`, `hister`, `wombat`) or connection
141
141
  aliases with `"type": "<template>"`. Unknown names are skipped with an
142
142
  `unknown-datasource-skill` warning; they do not fail config resolution.
143
143
  `scope` narrows a query to a sub-path as ordinary filtering. Tags are
@@ -94,7 +94,7 @@ approval before indexing:
94
94
  | Windows | Documents, Downloads, Desktop shell folders | OneDrive document roots, user-named project docs |
95
95
 
96
96
  Supported parsed formats are `md`, `markdown`, `txt`, `text`, `pdf`, `docx`,
97
- `pptx`, `xlsx`, `xls`, `hwp`, `hwpx`, and `eml`. OCR for `jpg`, `jpeg`, `png`, `bmp`,
97
+ `pptx`, `xlsx`, `xlsm`, `xlsb`, `xls`, `ods`, `csv`, `tsv`, `hwp`, `hwpx`, and `eml`. OCR for `jpg`, `jpeg`, `png`, `bmp`,
98
98
  and `tiff` is optional (`parserOptions.ocr.enabled`). Do not present legacy
99
99
  `.doc` as a supported parsed format.
100
100
 
@@ -162,8 +162,9 @@ workspace still answers, but without MinSync/Jikji evidence.
162
162
 
163
163
  Refresh (never a query) auto-installs the binaries: MinSync installs a verified
164
164
  GitHub release into `<workspace>/.autorag/bin` (`minSync.autoInstall` defaults
165
- to true), and Jikji installs `jikji-cli` through cargo (`jikji.autoInstall`
166
- defaults to true; requires the Rust toolchain). Set `"autoInstall": false` only
165
+ to true), and Jikji installs the pinned `jikji-cli` 0.2.1 through cargo (`jikji.autoInstall`
166
+ defaults to true; requires the Rust toolchain). A `jikji` older than 0.2.0 on PATH
167
+ or in the cache is skipped and replaced by that build on refresh. Set `"autoInstall": false` only
167
168
  when managing the binary yourself. Refresh is incremental: MinSync syncs only
168
169
  changed parsed mirrors, and `jikji prepare` reuses unchanged documents. Jikji
169
170
  roots prepare one at a time (they share one central SQLite file that rejects
@@ -178,8 +179,11 @@ with the `JIKJI_DATA_DIR` environment variable). The only file `refresh` adds
178
179
  inside an approved document root is the `.jikji_agent_map.md` guide. Tell the
179
180
  user that before the first `refresh`, and never delete or edit Jikji's central
180
181
  store by hand; removing it only forces a full Jikji re-prepare on the next
181
- refresh. Older Jikji versions wrote a `<root>/.jikji` directory instead; such a
182
- leftover is now unused and safe for the user to delete.
182
+ refresh. Jikji 0.1.x wrote a `<root>/.jikji` directory instead; such a
183
+ leftover is now unused and safe for the user to delete. Jikji 0.2.0 and newer
184
+ refuse to index a root nested inside, or containing, an already indexed root
185
+ (`prepare root overlaps existing indexed root`), which is why overlapping search
186
+ roots are collapsed to the outermost one before Jikji runs.
183
187
 
184
188
  Exact duplicate exclusion is enabled by default. AutoRAG invokes the external
185
189
  `dupey` CLI before parsed-mirror indexing, keeps the newest filesystem copy for
@@ -231,10 +235,11 @@ endpoint POSTs `{baseUrl}/embed`; `openai:` with no `baseUrl` means
231
235
  value. `dimension` and `batchSize` must be positive integers, and `dimension`
232
236
  must match the endpoint (the local default is 1024).
233
237
 
234
- ### Jev routing and question decomposition (on by default)
238
+ ### Jev decision model (required) and question decomposition (on by default)
235
239
 
236
- Leave both enabled. They are the recommended setup: they make simple questions
237
- fast and multi-part questions thorough.
240
+ Jev is required: AutoRAG cannot answer without it. Decomposition is on by
241
+ default; leave it enabled. Together they make simple questions fast and
242
+ multi-part questions thorough.
238
243
 
239
244
  - **Jev** (`jev`, default `{ "backend": "dazzi" }` when signed in to Dazzi
240
245
  Models, else `{ "backend": "openrouter" }`; model `jev-1.13` on Dazzi,
@@ -263,10 +268,13 @@ fast and multi-part questions thorough.
263
268
 
264
269
  Both need a credential for the chosen backend: a Dazzi Models `/login` (check with
265
270
  `autorag models list --provider dazzi --available`), or the user's
266
- `OPENROUTER_API_KEY` (`test -n "$OPENROUTER_API_KEY"`, never print it). Tell the
267
- user Jev routing is on. Without a credential, routing falls back to a single
268
- local search and the run always verifies, so searches still work. A
269
- `query-route-fallback` diagnostic (`autorag search --debug`) shows that state.
271
+ `OPENROUTER_API_KEY` (`test -n "$OPENROUTER_API_KEY"`, never print it). The first
272
+ question of every session checks that Jev answers; without a working credential
273
+ the search stops with `[decision-model-unavailable]` (`autorag search` exits 3,
274
+ and `--json` carries `code`, `reason`, `backend`, `model`). Read `reason`:
275
+ `missing-credential` means set the key or sign in, `unauthorized` means replace
276
+ the key, `model-not-found` means fix `jev.model`, `payment-required` means the
277
+ backend account is out of credit, `unreachable` and `rate-limited` mean retry.
270
278
 
271
279
  ```json
272
280
  {
@@ -277,14 +285,24 @@ local search and the run always verifies, so searches still work. A
277
285
 
278
286
  `autorag init` writes these defaults into new configs. To change them:
279
287
 
280
- - Jev backend: `"backend": "typesafe"` (`TYPESAFE_API_KEY`) or `"vercel"`
281
- (`AI_GATEWAY_API_KEY`).
288
+ - Jev backend and model: any provider with decision models, e.g.
289
+ `{ "backend": "openai", "model": "gpt-6-luna" }` (`OPENAI_API_KEY`),
290
+ `{ "backend": "openrouter", "model": "openai/gpt-6-luna-decisions" }`,
291
+ `"typesafe"` (`TYPESAFE_API_KEY`), `"vercel-ai-gateway"` (`AI_GATEWAY_API_KEY`),
292
+ `"cloudflare-workers-ai"`, or `"opencode"`. List the choices with
293
+ `autorag models list --decision --available`; never invent a model id.
294
+ - sLLM fallback: when Jev is unavailable, the session runs on a small chat model
295
+ (default: the decomposition model) and warns `decision-model-fallback`. Set the
296
+ model with `"jev": { "sllm": { "model": { "provider": …, "id": … } } }`, or
297
+ `"fallback": false` to fail instead.
298
+ - sLLM as the decision model: `"jev": { "backend": "sllm" }` decides with the
299
+ small chat model alone and never calls Jev (no Jev credential needed).
282
300
  - Decomposition model: any catalog `provider`/`id`, with the same fields as
283
301
  the top-level `model`.
284
302
  - `"queryDecomposition": false` decomposes with the search model itself.
285
- - `"jev": false` turns routing off entirely. Do this only when the user
286
- explicitly opts out, for example because questions must never leave the
287
- machine (Jev and decomposition send the question text to OpenRouter).
303
+ - Jev cannot be turned off: `"jev": false` and `jev.enabled` are config errors.
304
+ When questions must never leave the machine, AutoRAG is not usable; say so
305
+ instead of editing the config (Jev sends the question text to its backend).
288
306
 
289
307
  When a user asks the running agent itself to change its settings (switch the
290
308
  default model, add a provider, check that a provider works), Jev's `config`
@@ -305,12 +323,13 @@ resolution.
305
323
  | Field | Default | Controls |
306
324
  |---|---|---|
307
325
  | `mergedEvidenceCeiling` | 500 | `search_all_documents` / model-free merge ceiling when the model omits `topK` |
308
- | `singleDatasourceTopK` | 50 | `search_datasource_*` merge default when the model omits `topK` |
326
+ | `singleDatasourceTopK` | 20 | `search_datasource_*` merge default when the model omits `topK` |
309
327
  | `minSyncTopK` | 50 | MinSync semantic default `topK` |
310
328
  | `minSyncScopedQueryTopK` | 100 | MinSync fetch cap applied when a scope narrows the query |
311
329
  | `toolDescriptionInstanceScopes` | 8 | Instance scopes listed in one datasource tool description |
312
- | `prefetch.jikjiTopK` | 30 | Jikji find candidate count |
313
- | `prefetch.minSyncTopK` | 100 | MinSync retrieve candidate count |
330
+ | `prefetch.jikjiTopK` | 10 (30 with a rerank model) | Jikji find candidate count |
331
+ | `prefetch.minSyncTopK` | 10 (100 with a rerank model) | MinSync retrieve candidate count |
332
+ | `prefetch.datasourceTopK` | 10 (20 with a rerank model) | Hits requested from each selected datasource |
314
333
  | `prefetch.jikjiPathLimit` | 100 | Max Jikji answer paths rendered into the baseline |
315
334
  | `prefetch.sectionLimit` | 100 | Max results rendered per baseline section |
316
335
 
@@ -326,14 +345,19 @@ resolution.
326
345
  The baseline renders each prefetched result's full chunk content (not a
327
346
  per-result excerpt) so the model can see where the hit came from. Bound baseline
328
347
  size with `prefetch.sectionLimit` / `prefetch.minSyncTopK` /
329
- `prefetch.jikjiPathLimit`, not with a per-result content truncation.
348
+ `prefetch.datasourceTopK` / `prefetch.jikjiPathLimit`, not with a per-result
349
+ content truncation.
350
+
351
+ The `prefetch.*TopK` defaults depend on reranking. Reranking is opt-in and runs
352
+ only when `rerank.model` names a model; then the pool is wide (Jikji 30, MinSync
353
+ 100, each datasource 20) and the reranker picks the best. Without it (the
354
+ default) each source contributes its own top 10 to the first answer. An explicit
355
+ value always wins.
330
356
 
331
357
  Ingest caps are trusted connector options under `datasources.<name>.connector`
332
358
  and are never settable from model/tool arguments: `maxDocuments`,
333
- `maxItemsPerFeed`, and `maxContentChars` (RSS);
334
- `maxDocuments`, `maxContentChars`, `maxBytesPerFile`, `concurrency`,
335
- `bandwidthLimit`, and `dryRun` (cloud-drive/rclone). `maxContentChars` defaults
336
- to 20000 for RSS and 100000 for cloud-drive.
359
+ `maxItemsPerFeed`, and `maxContentChars` (RSS). `maxContentChars` defaults to
360
+ 20000 for RSS.
337
361
 
338
362
  ### Refresh safety limits
339
363
 
@@ -365,7 +389,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
365
389
 
366
390
  1. Probe every datasource for setup feasibility before asking the user
367
391
  anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
368
- `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`, `lark-cli`) and its
392
+ `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `wombat`, `lark-cli`) and its
369
393
  local store or archive is present. CLI-backed datasources own their own
370
394
  archive, index, and authentication, so environment credentials (such as bot
371
395
  tokens) are never required or checked for them. Non-CLI connectors
@@ -387,8 +411,8 @@ setup. Configure datasources directly in trusted config, wizard-style:
387
411
 
388
412
  Datasource skills belong in trusted config. Builtin
389
413
  template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
390
- `clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`,
391
- `mailcrawl`, `obsidian`, `rss`, `lark`, and `hister`. Config keys may be connection
414
+ `clawgallery`, `notion`, `github`, `github-gist`,
415
+ `mailcrawl`, `obsidian`, `rss`, `lark`, `hister`, and `wombat`. Config keys may be connection
392
416
  aliases with `"type": "<template>"`. Unknown names are skipped with an
393
417
  `unknown-datasource-skill` warning; they do not fail config resolution.
394
418
  `scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.
@@ -398,9 +422,8 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
398
422
  "datasources": {
399
423
  "github": { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
400
424
  "github-gist": { "connector": { "tokenEnv": "GITHUB_TOKEN" } },
401
- "google-drive": { "type": "cloud-drive", "connector": { "provider": "google-drive", "remote": "gdrive:" } },
402
- "archive-drive": { "type": "cloud-drive", "connector": { "remote": "archive:" } },
403
425
  "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
426
+ "wombat": { "connector": { "binaryPath": "wombat", "agents": ["codex", "claude-code"] } },
404
427
  "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
405
428
  "rss": { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
406
429
  }
@@ -1,2 +0,0 @@
1
- export { RcloneConnector, type RcloneConnectorOptions, type RcloneRunner, type RcloneRunResult, } from "./rclone-connector.js";
2
- export { CLOUD_DRIVE_SKILL_DEFINITION, CloudDriveSkill, type CloudDriveSkillOptions, } from "./skill.js";
@@ -1,49 +0,0 @@
1
- /**
2
- * Provider-neutral incremental rclone mirror.
3
- *
4
- * `lsjson` inventories a remote, a workspace manifest computes its diff, and
5
- * only added/changed indexable files are copied into a managed mirror. The
6
- * manifest is published only after every changed file is available, so search
7
- * can continue using the previous completed snapshot after an interrupted run.
8
- */
9
- import type { DefaultParserRegistryOptions } from "../../../parser/defaults.js";
10
- import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
11
- export interface RcloneRunResult {
12
- readonly ok: boolean;
13
- readonly stdout: string;
14
- readonly stderr: string;
15
- readonly code: number | null;
16
- }
17
- export type RcloneRunner = (args: readonly string[], timeoutMs: number) => Promise<RcloneRunResult>;
18
- export interface RcloneConnectorOptions {
19
- readonly binaryPath?: string;
20
- readonly remote?: string;
21
- readonly provider?: string;
22
- readonly workspaceRoot?: string;
23
- readonly instanceId?: string;
24
- readonly skillName?: string;
25
- readonly extensions?: readonly string[];
26
- readonly include?: readonly string[];
27
- readonly exclude?: readonly string[];
28
- readonly exportFormats?: string;
29
- readonly maxDocuments?: number;
30
- readonly maxBytesPerFile?: number;
31
- /** Characters kept per document body. Default 100000. */
32
- readonly maxContentChars?: number;
33
- readonly concurrency?: number;
34
- readonly bandwidthLimit?: string;
35
- readonly dryRun?: boolean;
36
- readonly timeoutMs?: number;
37
- readonly runner?: RcloneRunner;
38
- /** Explicit operator-owned rclone.conf path, passed read-only. */
39
- readonly configPath?: string;
40
- readonly parserOptions?: DefaultParserRegistryOptions;
41
- }
42
- export declare class RcloneConnector implements DatasourceConnector {
43
- private readonly options;
44
- private readonly runner;
45
- constructor(options?: RcloneConnectorOptions);
46
- fetch(): Promise<ConnectorFetchResult>;
47
- private fetchIncremental;
48
- private fetchWithoutMirror;
49
- }
@@ -1,14 +0,0 @@
1
- import type { DatasourceConnector } from "../../connector.js";
2
- import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
3
- import { type RcloneConnectorOptions } from "./rclone-connector.js";
4
- export declare const CLOUD_DRIVE_SKILL_DEFINITION: ConnectorSkillDefinition;
5
- export interface CloudDriveSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
6
- /** Unique model-visible datasource name for this configured connection. */
7
- readonly skillName?: string;
8
- readonly provider?: "google-drive" | "onedrive" | "icloud" | string;
9
- readonly connector?: DatasourceConnector;
10
- readonly connectorOptions?: RcloneConnectorOptions;
11
- }
12
- export declare class CloudDriveSkill extends ConnectorDatasourceSkill {
13
- constructor(options?: CloudDriveSkillOptions);
14
- }
@@ -1,17 +0,0 @@
1
- /** What the probe reads, injectable so every platform's layout can be checked from any host. */
2
- export interface KiwiProbeEnvironment {
3
- readonly env: NodeJS.ProcessEnv;
4
- readonly platform: NodeJS.Platform;
5
- readonly home: string | undefined;
6
- readonly exists: (path: string) => boolean;
7
- }
8
- /**
9
- * MinSync's `ko` tokenizer loads the Kiwi native library and its base model at run time (through `kiwi-rs`); neither
10
- * ships with MinSync. Without them every sync that needs Korean tokenization fails, so AutoRAG checks first.
11
- *
12
- * This mirrors the filesystem part of `kiwi-rs`'s discovery (`KIWI_LIBRARY_PATH` / `KIWI_MODEL_PATH`, then the
13
- * per-user `~/.local/kiwi` install and the well-known system directories). Names left to the OS loader search path
14
- * are not probed, so a library found only that way reads as missing; point `KIWI_LIBRARY_PATH` at it to fix that.
15
- * kiwi-rs treats an explicit path as authoritative, and so does this probe.
16
- */
17
- export declare function kiwiAvailable(environment?: KiwiProbeEnvironment): boolean;
@@ -1,18 +0,0 @@
1
- /**
2
- * The BM25 tokenizer presets AutoRAG sets in MinSync's `[lexical].language`. MinSync also has `ko`, `ja`, `zh` and
3
- * `ar`, which AutoRAG never selects.
4
- */
5
- export type MinSyncLexicalLanguage = "simple" | "multilingual";
6
- /**
7
- * The tokenizer AutoRAG always asks MinSync for, whatever the global `languages` setting says.
8
- *
9
- * `simple` splits on whitespace only, so a Korean word with a particle (인증서를) never matches its stem (인증서).
10
- * `ko` fixes that by indexing only Kiwi's morphemes, but then a compound (연구목표) exists only as 연구 and 목표 and an
11
- * exact-term query loses to documents that merely contain both pieces. `multilingual` indexes the original
12
- * whitespace-separated words as well as every analyzer's tokens, so exact terms and inflected forms both match. On
13
- * a sample of real Korean documents (60 stem queries with inflected forms, 40 exact-term queries, BM25 only) it had the
14
- * best recall on both and kept most of `simple`'s top-5 precision on exact terms, where `ko` lost most of it.
15
- *
16
- * The preset needs the Kiwi native library; see `kiwi.ts`.
17
- */
18
- export declare const MINSYNC_LEXICAL_LANGUAGE: MinSyncLexicalLanguage;