@autorag/librarian 2.13.4 → 2.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -0
- package/README.md +50 -18
- package/dist/agent/agent.d.ts +111 -40
- package/dist/agent/cloud-release-extension.d.ts +43 -0
- package/dist/agent/decision-backends.d.ts +49 -0
- package/dist/agent/decision-model-check.d.ts +92 -0
- package/dist/agent/dupey-tool.d.ts +2 -0
- package/dist/agent/everything-search-tool.d.ts +1 -0
- package/dist/agent/fsearch-search-tool.d.ts +1 -0
- package/dist/agent/index.d.ts +4 -1
- package/dist/agent/jev-extension.d.ts +27 -32
- package/dist/agent/jikji-find-tool.d.ts +5 -0
- package/dist/agent/name-candidates.d.ts +88 -0
- package/dist/agent/permission-gate.d.ts +2 -5
- package/dist/agent/query-routing.d.ts +1 -1
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/agent/sllm-decision.d.ts +34 -0
- package/dist/cli/commands/duplicates.d.ts +3 -2
- package/dist/cli/commands/models.d.ts +2 -0
- package/dist/cli/commands/search.d.ts +2 -0
- package/dist/cli/config.d.ts +49 -30
- package/dist/cli/index.js +13522 -10169
- package/dist/cli/setup.d.ts +27 -1
- package/dist/cloud/jev-backend.d.ts +26 -4
- package/dist/core.js +6079 -2840
- package/dist/datasource/index.d.ts +2 -2
- package/dist/datasource/skills/obsidian/index.d.ts +1 -1
- package/dist/datasource/skills/obsidian/methods.d.ts +1 -1
- package/dist/datasource/skills/obsidian/skill.d.ts +1 -0
- package/dist/datasource/skills/obsidian/vault-index.d.ts +1 -1
- package/dist/datasource/skills/wombat/client.d.ts +17 -0
- package/dist/datasource/skills/wombat/index.d.ts +4 -0
- package/dist/datasource/skills/wombat/methods.d.ts +15 -0
- package/dist/datasource/skills/wombat/skill.d.ts +38 -0
- package/dist/datasource/skills/wombat/types.d.ts +89 -0
- package/dist/dupey/cli.d.ts +21 -1
- package/dist/dupey/filter.d.ts +8 -1
- package/dist/dupey/index.d.ts +1 -0
- package/dist/dupey/scan-plan.d.ts +23 -0
- package/dist/embedding/index.d.ts +3 -1
- package/dist/everything/client.d.ts +1 -2
- package/dist/filesystem/cloud-placeholder.d.ts +313 -21
- package/dist/filesystem/shared-exclusive-queue.d.ts +16 -0
- package/dist/fsearch/client.d.ts +3 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +6184 -2905
- package/dist/jikji/answer-pack.d.ts +7 -1
- package/dist/jikji/client.d.ts +28 -14
- package/dist/jikji/diagnostics.d.ts +16 -2
- package/dist/jikji/index.d.ts +3 -3
- package/dist/jikji/installer.d.ts +11 -5
- package/dist/jikji/retry-command.d.ts +10 -0
- package/dist/jikji/types.d.ts +47 -1
- package/dist/jikji/version.d.ts +20 -0
- package/dist/mcp/index.js +6154 -2902
- package/dist/mcp/server.d.ts +3 -0
- package/dist/minsync/binary.d.ts +10 -5
- package/dist/minsync/client.d.ts +7 -10
- package/dist/minsync/embedder-config.d.ts +11 -10
- package/dist/minsync/errors.d.ts +3 -0
- package/dist/minsync/index.d.ts +1 -1
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/method.d.ts +1 -0
- package/dist/minsync/types.d.ts +0 -9
- package/dist/minsync/version.d.ts +20 -0
- package/dist/mirror/cloud-fingerprints.d.ts +30 -0
- package/dist/mirror/cloud.d.ts +79 -0
- package/dist/mirror/index.d.ts +4 -0
- package/dist/mirror/sync.d.ts +79 -1
- package/dist/parser/csv.d.ts +26 -0
- package/dist/parser/defaults.d.ts +1 -1
- package/dist/parser/index.d.ts +2 -0
- package/dist/parser/kordoc.d.ts +6 -3
- package/dist/parser/parse-worker.js +668 -28
- package/dist/parser/spreadsheet-render.d.ts +53 -0
- package/dist/parser/spreadsheet.d.ts +27 -0
- package/dist/profiles/loop-guard.d.ts +8 -2
- package/dist/profiles/retrieval-tools.d.ts +2 -1
- package/dist/profiles.js +13520 -10280
- package/dist/retrieval/skip.d.ts +1 -1
- package/dist/retrieval/types.d.ts +1 -1
- package/package.json +3 -3
- package/skills/autorag/SKILL.md +7 -4
- package/skills/autorag-doctor/SKILL.md +8 -3
- package/skills/autorag-lite-setup/SKILL.md +4 -4
- package/skills/autorag-setup/SKILL.md +53 -30
- package/dist/datasource/skills/cloud-drive/index.d.ts +0 -2
- package/dist/datasource/skills/cloud-drive/rclone-connector.d.ts +0 -49
- package/dist/datasource/skills/cloud-drive/skill.d.ts +0 -14
- package/dist/minsync/kiwi.d.ts +0 -17
- package/dist/minsync/lexical-language.d.ts +0 -18
package/NOTICE
CHANGED
|
@@ -78,6 +78,9 @@ License: Apache-2.0
|
|
|
78
78
|
typebox@1.3.37
|
|
79
79
|
License: MIT
|
|
80
80
|
|
|
81
|
+
xlsx@0.20.3
|
|
82
|
+
License: Apache-2.0
|
|
83
|
+
|
|
81
84
|
Bundled Windows binaries (vendor/everything, extracted only on Windows):
|
|
82
85
|
Everything 1.4.1.1032 portable (voidtools, David Carpenter)
|
|
83
86
|
License: MIT (closed source); includes PCRE (BSD-3-Clause)
|
package/README.md
CHANGED
|
@@ -187,11 +187,11 @@ AutoRAG Agent:
|
|
|
187
187
|
|
|
188
188
|
Three principles drive every design decision in AutoRAG Agent:
|
|
189
189
|
|
|
190
|
-
1. **Never migrate your data to search it.** Traditional RAG systems force you to upload, ETL, and duplicate your files into a centralized vector database. AutoRAG Agent federates your data **in place**, querying CLI-native stores (`lazykatok`, `discrawl`, `slacrawl`, `mailcrawl
|
|
190
|
+
1. **Never migrate your data to search it.** Traditional RAG systems force you to upload, ETL, and duplicate your files into a centralized vector database. AutoRAG Agent federates your data **in place**, querying CLI-native stores (`lazykatok`, `discrawl`, `slacrawl`, `mailcrawl`) where your data already lives. Results retain opaque, source-native identities (`/kakao/...`, `/slack/...`) that preserve local access control and privacy. *(See our [Competitive Landscape Study](docs/competitive-landscape-2026-09.md) on why in-place federation is the durable differentiator).*
|
|
191
191
|
|
|
192
|
-
2. **Just works — no RAG degree required.** No pipeline tuning, no vector-DB maintenance, and no remote embedding keys required. AutoRAG Agent automatically manages [MinSync](docs/minsync-setup.md) for incremental Change Data Capture (CDC) chunking and provides a local [embedding backend](docs/minsync-setup.md#embedding-backends) out of the box with zero external telemetry. A [remote rerank model](docs/rerank.md)
|
|
192
|
+
2. **Just works — no RAG degree required.** No pipeline tuning, no vector-DB maintenance, and no remote embedding keys required. AutoRAG Agent automatically manages [MinSync](docs/minsync-setup.md) for incremental Change Data Capture (CDC) chunking and provides a local [embedding backend](docs/minsync-setup.md#embedding-backends) out of the box with zero external telemetry. A [remote rerank model](docs/rerank.md) is an opt-in extra: it runs only when you name one in `rerank.model`, and otherwise the first answer is built from each source's own top 10.
|
|
193
193
|
|
|
194
|
-
3. **Fast by design.** Rather than coordinating slow multi-agent hierarchies, a single configured model owns the entire retrieval, direct-read, and curation loop. Local CDC chunks (
|
|
194
|
+
3. **Fast by design.** Rather than coordinating slow multi-agent hierarchies, a single configured model owns the entire retrieval, direct-read, and curation loop. Local CDC chunks (SPLADE sparse, vector, and hybrid modes) deliver rapid, low-latency turnaround across multi-turn research queries.
|
|
195
195
|
|
|
196
196
|
---
|
|
197
197
|
|
|
@@ -201,9 +201,9 @@ AutoRAG Agent orchestrates five integrated subsystems:
|
|
|
201
201
|
|
|
202
202
|
1. **Retrieval Memory (`check_memory`):** After each answer, cited evidence Jev judges to genuinely support it is stored in `~/.autorag/memory.json`. Later searches receive this conversation's evidence, similar past questions, and long-term insights as **advisory reference** — it never reorders results or overrides current evidence. See [docs/retrieval-memory.md](docs/retrieval-memory.md).
|
|
203
203
|
2. **Pluggable Multi-Method Retrieval:**
|
|
204
|
-
- **
|
|
204
|
+
- **SPLADE Sparse Search:** Learned-sparse keyword ranking via MinSync's KURE-Sparse-nano encoder over parsed markdown mirrors; it replaced BM25 in MinSync 0.6 and matches Korean inflected forms (`인증서를` → `인증서`) with no tokenizer library. Exact identifiers among many look-alike files rank less reliably than with BM25. See [docs/minsync-setup.md](docs/minsync-setup.md#sparse-retrieval-splade).
|
|
205
205
|
- **Semantic Vector Search:** Dense vector retrieval over CDC chunks via the shared MinSync embedder.
|
|
206
|
-
- **Hybrid Search:**
|
|
206
|
+
- **Hybrid Search:** MinSync fuses sparse and vector ranking over the same chunks.
|
|
207
207
|
- **Jikji Find-First Discovery:** Local CLI-backed fast discovery answer packs.
|
|
208
208
|
- **Everything File-Name Search (Windows):** The bundled [voidtools Everything](https://www.voidtools.com/) indexes file and folder names under your search roots so the agent finds files by name, extension, path, size, or date instantly (`everything_search`).
|
|
209
209
|
- **FSearch File-Name Search (macOS/Linux):** [fsearch-cli](https://github.com/NomaDamas/fsearch-mac) (FSearch) keeps a live, per-workspace name index of your search roots so the agent finds files by name, extension, path, size, or date instantly (`fsearch_search`); without fsearch-cli installed it degrades to a slow filesystem walk.
|
|
@@ -260,7 +260,7 @@ bun add @autorag/librarian
|
|
|
260
260
|
### System Prerequisites
|
|
261
261
|
|
|
262
262
|
- **No Java required:** Document parsing (HWP/HWPX/HWPML, PDF, DOCX, XLSX/XLS) runs in-process through [`kordoc`](https://github.com/chrisryugj/kordoc).
|
|
263
|
-
- **Rust Toolchain (Optional):** Automatically compiles Jikji (`jikji-cli
|
|
263
|
+
- **Rust Toolchain (Optional):** Automatically compiles Jikji (`jikji-cli`, pinned to 0.2.1) if installed. A `jikji` older than 0.2.0 on your PATH is skipped and replaced by the pinned build on the next refresh.
|
|
264
264
|
- **MinSync:** Automatically downloaded and installed into `<workspace>/.autorag/bin` on first run.
|
|
265
265
|
- **Everything (Windows only, bundled):** The package ships the portable Everything 1.4.1.1032 and its ES 1.1.0.38 CLI (x64 and ARM64, SHA-256 pinned). On Windows, AutoRAG extracts them into `<workspace>/.autorag/everything/` and runs a private, user-level instance that indexes only your configured search roots: no administrator rights, no Everything service, no whole-drive scan, no HTTP/ETP server, and no change to any Everything you already run. Set `"everything": false` in the config to turn it off. macOS and Linux are not affected.
|
|
266
266
|
- **filee-cli (file conversion, automatic):** `autorag setup` and the TUI download the pinned filee-cli release (~85 MB, SHA-256 verified) into `<AUTORAG_HOME>/tools/filee-cli/<version>/`, so the agent can convert between 255 formats offline with `convert_files`. A `filee`/`filee-cli` already on `PATH` is used instead. Set `"filee": { "autoInstall": false }` to forbid the download or `"filee": { "enabled": false }` to remove the tools. Filee fetches its own engines (FFmpeg, Pandoc, LibreOffice, calibre, Ghostscript; hundreds of MB, only when a conversion needs one) on first use.
|
|
@@ -279,7 +279,7 @@ autorag init --search-paths ~/Documents/research
|
|
|
279
279
|
# Korean + English is the default; set it explicitly for other corpora
|
|
280
280
|
autorag init --search-paths ~/Documents/research --languages ja,en
|
|
281
281
|
|
|
282
|
-
# Index documents (parses PDFs/Markdown, builds
|
|
282
|
+
# Index documents (parses PDFs/Markdown, builds MinSync sparse and dense vectors)
|
|
283
283
|
autorag refresh
|
|
284
284
|
|
|
285
285
|
# Check indexing health
|
|
@@ -314,7 +314,9 @@ falling back to `["ko", "en"]`. Supported tags: `ko`, `en`, `ja`, `zh-hans`,
|
|
|
314
314
|
|
|
315
315
|
| Extension | Parser |
|
|
316
316
|
|---|---|
|
|
317
|
-
| `.hwp` `.hwpx` `.hml` `.hwpml` `.pdf` `.docx`
|
|
317
|
+
| `.hwp` `.hwpx` `.hml` `.hwpml` `.pdf` `.docx` | `kordoc` (nested tables, no Java) |
|
|
318
|
+
| `.xlsx` `.xlsm` `.xlsb` `.xls` `.ods` | built-in spreadsheet reader (SheetJS) |
|
|
319
|
+
| `.csv` `.tsv` | built-in CSV reader (CP949/EUC-KR aware) |
|
|
318
320
|
| `.pptx` | built-in PPTX reader |
|
|
319
321
|
| `.eml` | built-in mail reader |
|
|
320
322
|
| `.txt` `.text` `.md` `.markdown` | plain text (CP949/EUC-KR aware) |
|
|
@@ -322,6 +324,34 @@ falling back to `["ko", "en"]`. Supported tags: `ko`, `en`, `ja`, `zh-hans`,
|
|
|
322
324
|
|
|
323
325
|
`.bmp` and `.tiff` are not parsed: kordoc rejects both as unsupported.
|
|
324
326
|
|
|
327
|
+
The spreadsheet reader writes one line per row. Each line names the sheet, the
|
|
328
|
+
cell range and the table title, and pairs every value with its column label, so
|
|
329
|
+
a search hit always says what a number is:
|
|
330
|
+
|
|
331
|
+
```text
|
|
332
|
+
[예산총괄!A4:F4 · 2026년 연구개발비 편성표 (단위: 천원)] 구분: 직접비 | 비목: 인건비 | 편성액: 742,000 | 비율(%): 32.1%
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
Values are what the spreadsheet shows: cached formula results with their number
|
|
336
|
+
formats (`742,000`, `32.1%`, dates). A format that scales the stored number keeps
|
|
337
|
+
that number too (`742천원 (742000)`), and a formula saved without a result shows
|
|
338
|
+
its formula (`=SUM(B2:B3)`). Titles above a table become its caption, merged
|
|
339
|
+
group headers become `상반기 > 1월`, several tables on one sheet are kept apart,
|
|
340
|
+
and a row longer than a search chunk is split into lines that repeat its first
|
|
341
|
+
value. A first row counts as the header only with clear evidence (a merged group
|
|
342
|
+
header, or at least three rows below it with a mostly numeric, date or formula
|
|
343
|
+
column); otherwise rows are kept exactly as written, so a form's label/value
|
|
344
|
+
cells are never relabelled. Sheets are read up to 100,000 rows each; a longer
|
|
345
|
+
sheet is indexed partially with a `parser-warning` diagnostic.
|
|
346
|
+
|
|
347
|
+
`.csv` and `.tsv` files produce the same lines, with the file name in place of
|
|
348
|
+
the sheet name. Bytes decode as UTF-8 (with or without BOM) or CP949/EUC-KR,
|
|
349
|
+
quoted fields keep separators, line breaks and doubled quotes, and values stay
|
|
350
|
+
exactly as written (`00123` and `=1+2` are not converted). `.tsv` is
|
|
351
|
+
tab-separated; a `.csv` follows Excel's `sep=;` first line when present and
|
|
352
|
+
otherwise picks the separator (`,` `;` tab `|`) that splits its first lines
|
|
353
|
+
most consistently.
|
|
354
|
+
|
|
325
355
|
### Making photos and scans searchable
|
|
326
356
|
|
|
327
357
|
Image files and scanned PDF pages become searchable through one switch,
|
|
@@ -348,7 +378,7 @@ explicit `parserOptions.kordoc.ocr` overrides the language-based choice.
|
|
|
348
378
|
|
|
349
379
|
### Bounding and cancelling a parse
|
|
350
380
|
|
|
351
|
-
`kordoc` (PDF, HWP/HWPX, DOCX,
|
|
381
|
+
`kordoc` (PDF, HWP/HWPX, DOCX, images), the spreadsheet reader, `.pptx` and `.eml` parse in a worker
|
|
352
382
|
thread during a refresh, so one corrupt or hostile file cannot hang or crash the run. A
|
|
353
383
|
`Promise.race` deadline cannot stop a synchronous parse; terminating the worker can.
|
|
354
384
|
|
|
@@ -480,7 +510,7 @@ behaves as before.
|
|
|
480
510
|
| `autorag search "<query>"` | Run the librarian agent to curate structured answers |
|
|
481
511
|
| `autorag status` | Inspect corpus freshness, indexing status, and vector readiness |
|
|
482
512
|
| `autorag health` | Check model provider authentication, token validity, and API reachability |
|
|
483
|
-
| `autorag models list` | List chat models the pi runtime can resolve (built-ins, `models.json`, custom/extension providers) with provider auth status; never prints credential values |
|
|
513
|
+
| `autorag models list` | List chat models the pi runtime can resolve (built-ins, `models.json`, custom/extension providers) with provider auth status; `--decision` lists decision models (`jev.backend`/`jev.model` candidates) instead; never prints credential values |
|
|
484
514
|
| `autorag update-check` | Compare the running `autorag` against the published npm version (also runs on `autorag tui` launch) |
|
|
485
515
|
| `autorag tui` | Open Pi's interactive librarian TUI (`/login`, `/model`, `/resume`, …); the agent can stop and ask you a multiple-choice or open question through the [`ask_user` tool](docs/ask-user.md); `--telegram` / `--whatsapp` / `--discord` add the Telegram / WhatsApp / Discord remote-control bridges; `--permission-mode ask\|auto\|full` (Shift+Tab, `/permissions`) sets the [approval gate for file changes](docs/permissions.md) |
|
|
486
516
|
| `autorag duplicates [DIR]` | Read-only scan for exact and near-duplicate document families with `dupey` |
|
|
@@ -653,7 +683,7 @@ AutoRAG Agent connects to external tools and communication platforms using dedic
|
|
|
653
683
|
|
|
654
684
|
| Datasource | Skill Alias | Backend / Driver | Storage & Privacy Model | Search Capabilities |
|
|
655
685
|
|---|---|---|---|---|
|
|
656
|
-
| **Local Documents** | `local` | Native filesystem & MinSync | Workspace-local parsed mirrors (`.autorag/`) |
|
|
686
|
+
| **Local Documents** | `local` | Native filesystem & MinSync | Workspace-local parsed mirrors (`.autorag/`) | Sparse (SPLADE), Semantic, Hybrid |
|
|
657
687
|
| **KakaoTalk** | `kakao` | [`lazykatok`](https://github.com/changeroa/lazykatok) CLI | Native KakaoTalk archive; zero direct DB access | BM25, Semantic, Hybrid |
|
|
658
688
|
| **Discord** | `discord` | [`discrawl`](https://github.com/openclaw/discrawl) CLI | Native SQLite archive; token-free wiretap mode | BM25 + native semantic hybrid ranking by default via a [read-only sidecar](docs/datasource-skills.md#discord-bm25-ranking); native FTS supplements lexical recall |
|
|
659
689
|
| **WhatsApp** | `whatsapp` | [`wacrawl`](https://github.com/openclaw/wacrawl) CLI | Local-first incremental archive + FTS5 | Lexical FTS5; BM25 + Semantic + Hybrid by default via [`sidecar`](docs/datasource-skills.md#hybrid-search-for-chat-crawlers-sidecar) |
|
|
@@ -661,10 +691,10 @@ AutoRAG Agent connects to external tools and communication platforms using dedic
|
|
|
661
691
|
| **Slack** | `slack` | [`slacrawl`](https://github.com/openclaw/slacrawl) CLI | Local workspace/channel/thread archive + FTS5 | Lexical FTS5; BM25 + Semantic + Hybrid by default via [`sidecar`](docs/datasource-skills.md#hybrid-search-for-chat-crawlers-sidecar) |
|
|
662
692
|
| **Notion** | `notion` | [`notcrawl`](https://github.com/openclaw/notcrawl) CLI | Local page/database/block archive + FTS5 | Lexical FTS5 |
|
|
663
693
|
| **Email Archives** | `mailcrawl` | [`mailcrawl`](https://github.com/NomaDamas/mailcrawl) CLI | Local Gmail, IMAP, Maildir, and `.eml`/`.mbox` mailbox backups (`--source files`) | BM25, Semantic, Hybrid |
|
|
694
|
+
| **Coding-Agent Sessions** | `wombat` | [`wombat`](https://github.com/NomaDamas/wombat) CLI (`cargo install wombat-sessions`) | Codex, Claude Code, OpenCode, pi, and omp conversations; thinking, tool output, and diffs are not indexed | Keyword (SPLADE), Semantic, Hybrid |
|
|
664
695
|
| **Obsidian Vaults** | `obsidian` | Indexed with MinSync in its own workspace (no external CLI) | Markdown notes, folder-scopable | Lexical (BM25), Semantic, Scoped |
|
|
665
696
|
| **GitHub** | `github` | GitHub REST API | In-memory fetched Issues and Pull Requests | Lexical, Scoped |
|
|
666
697
|
| **GitHub Gists** | `github-gist` | GitHub REST API | Incremental local index of own-account public and permitted secret Gist content/metadata; token not stored | Lexical (BM25), Local Semantic, Scoped |
|
|
667
|
-
| **Cloud Drives** | `cloud-drive` | [`rclone`](https://rclone.org) CLI | Google Drive (Tier-1), OneDrive, Dropbox, etc. | Incremental Mirror + BM25 |
|
|
668
698
|
| **Photos & Shots** | `clawgallery` | `clawgallery` CLI | Local screenshot and photo store | Hybrid OCR/Visual Search |
|
|
669
699
|
| **RSS / News** | `rss` | Native HTTP Poller | RSS 2.0 & Atom feeds (24h deduplication) | Lexical |
|
|
670
700
|
| **Browser History** | `hister` | [Hister](https://github.com/asciimoo/hister) server (AGPL-3.0-or-later, separate program, HTTP API) | Pages visited in Chrome/Chromium/Firefox, captured by Hister's own extension and indexed by Hister (web pages only; its local-file indexing is never used) | Lexical full-text; optional Semantic via Hister's own embedding endpoint |
|
|
@@ -733,9 +763,9 @@ the whole librarian loop runs on the hosted account:
|
|
|
733
763
|
|
|
734
764
|
`jev.backend: "dazzi"` posts Jev's native dialect to `<root>/v1/jev`
|
|
735
765
|
(default model `jev-1.13`); `rerank.provider: "dazzi"` reranks against
|
|
736
|
-
`<root>/v1/rerank
|
|
737
|
-
|
|
738
|
-
the
|
|
766
|
+
`<root>/v1/rerank`. Both use the stored `/login` credential. Reranking is opt-in and
|
|
767
|
+
has no default model: it runs only when `rerank.model` names one, and an explicit
|
|
768
|
+
`rerank.baseUrl` overrides the endpoint.
|
|
739
769
|
|
|
740
770
|
---
|
|
741
771
|
|
|
@@ -749,7 +779,7 @@ Deep dive into AutoRAG Agent's architecture, security, and integration guides:
|
|
|
749
779
|
- **[Telegram Remote Control](docs/telegram.md):** Ask the librarian from a Telegram chat through a running `autorag tui --telegram` session: setup, commands, and keeping it alive.
|
|
750
780
|
- **[Datasource Skills Reference](docs/datasource-skills.md):** Full configuration contracts, connection aliases, and connector options.
|
|
751
781
|
- **[Apple Calendar & Reminders via EventKit MCP](docs/eventkit-mcp.md):** macOS-only: read Calendar events and Reminders (Google/iCloud/Exchange accounts already in Calendar.app) by running `mcp-server-apple-events` beside `autorag-mcp`. Live, read-only; not indexed.
|
|
752
|
-
- **[Jev Decisions](docs/jev-decisions.md):** Jev
|
|
782
|
+
- **[Jev Decisions](docs/jev-decisions.md):** Jev, the **required** decision model, via Dazzi Models when signed in (OpenRouter otherwise), or any decision model in pi's catalog: `jev.backend` names the provider (`openrouter`, `openai`, `typesafe`, `vercel-ai-gateway`, `cloudflare-workers-ai`, `opencode`, or `dazzi`) and `jev.model` the model (e.g. `openai/gpt-6-luna-decisions` on OpenRouter, `gpt-6-luna` on OpenAI); `autorag models list --decision` lists them. The first question of every session checks that it answers; when it cannot, the session moves to an sLLM fallback (`jev.sllm.model`, the decomposition model by default, with a `decision-model-fallback` warning), and `jev.backend: "sllm"` makes that small chat LLM the decision model with no Jev call, and when that fails too it stops with `decision-model-unavailable` (exit 3 for `autorag search`). It routes each question to local search (everything needing more information, including the public web when the question needs recent public facts), `extra` (a knowledge answer or any other action such as committing code; one turn with every tool, no retrieval), `organize` (move, rename, convert, or sort files and folders), or its own `config` branch, and keeps a weakly-favored non-local verdict on local search through a 0.75 confidence floor. It also splits multi-part questions into up to five parallel searches (`dazzi/openai/gpt-6-luna` when signed in, else `openrouter/qwen/qwen3.7-flash`), picks which registered datasources to search before the fast answer, and ends the run after the fast answer when that answer is complete.
|
|
753
783
|
- **[Manual QA & Datasource Test Harnesses](docs/manual-qa-datasources.md):** Real-world testing guides for Discord, KakaoTalk, Slack, Notion, and email.
|
|
754
784
|
- **[Supply Chain Security & License Audits](docs/supply-chain.md):** Software bill of materials (SBOM) and dependency gate policies.
|
|
755
785
|
- **[Competitive Landscape Study](docs/competitive-landscape-2026-09.md):** In-depth analysis of why in-place federation outperforms centralized RAG.
|
|
@@ -781,7 +811,7 @@ AutoRAG Agent stands on the shoulders of fantastic open-source projects:
|
|
|
781
811
|
- **[Hister](https://github.com/asciimoo/hister)** by [asciimoo](https://github.com/asciimoo) and contributors — Self-hosted browser web-history search (AGPL-3.0-or-later). Never bundled, linked, copied, or modified: users run Hister as a separate program and AutoRAG only calls its public HTTP search API (mere aggregation). Hister is always named as the backend of the `hister` datasource.
|
|
782
812
|
- **[dupey](https://github.com/NomaDamas/dupey)** by [NomaDamas](https://github.com/NomaDamas) — Fast duplicate and near-duplicate document family detection.
|
|
783
813
|
- **[Filee](https://github.com/KnifeLemon/Filee)** by [KnifeLemon](https://github.com/KnifeLemon) — Offline conversion of 240+ file formats (MIT). Never bundled or linked: AutoRAG downloads the SHA-256-pinned [`filee-cli`](https://github.com/NomaDamas/filee-cli) release (NomaDamas' CLI-only redistribution, MIT) into `<AUTORAG_HOME>/tools/filee-cli/` on `autorag setup` / first conversion and spawns it as a separate process. An existing `filee`/`filee-cli` on `PATH` or `filee.binaryPath` wins. Filee downloads its own engines (FFmpeg, Pandoc, LibreOffice, …) on first use.
|
|
784
|
-
- **Federated CLI Authors:** External datasource tools [`lazykatok`](https://github.com/changeroa/lazykatok), [`discrawl`](https://github.com/openclaw/discrawl), [`mailcrawl`](https://github.com/NomaDamas/mailcrawl), [`wacrawl`](https://github.com/openclaw/wacrawl), [`telecrawl`](https://github.com/openclaw/telecrawl), [`slacrawl`](https://github.com/openclaw/slacrawl), [`notcrawl`](https://github.com/openclaw/notcrawl)
|
|
814
|
+
- **Federated CLI Authors:** External datasource tools [`lazykatok`](https://github.com/changeroa/lazykatok), [`discrawl`](https://github.com/openclaw/discrawl), [`mailcrawl`](https://github.com/NomaDamas/mailcrawl), [`wacrawl`](https://github.com/openclaw/wacrawl), [`telecrawl`](https://github.com/openclaw/telecrawl), [`slacrawl`](https://github.com/openclaw/slacrawl), and [`notcrawl`](https://github.com/openclaw/notcrawl).
|
|
785
815
|
- **[kordoc](https://github.com/chrisryugj/kordoc)** by [chrisryugj](https://github.com/chrisryugj) — HWP/HWPX/HWPML, PDF, DOCX and XLSX parsing with nested-table fidelity, used as AutoRAG's default document parser.
|
|
786
816
|
|
|
787
817
|
---
|
|
@@ -822,9 +852,11 @@ Common failures and their fix:
|
|
|
822
852
|
| A datasource errors during refresh | `datasource-index-failed` | run that CLI's own `doctor` |
|
|
823
853
|
| MinSync binary missing (search and refresh fail with `MinSync is required ...`) | — | `cargo install minsync`, or leave `minSync.autoInstall` on and re-run refresh |
|
|
824
854
|
| Jikji missing | `jikji-unavailable` | check the Rust toolchain, re-run refresh |
|
|
855
|
+
| Jikji older than 0.2.0 (the message names the version and the binary path) | `jikji-unsupported-version` | re-run `autorag refresh --method jikji` to install the pinned build, or `cargo install jikji-cli --version 0.2.1 --locked --force`; with `jikji.autoInstall: false` or `jikji.binaryPath`, upgrade it yourself |
|
|
825
856
|
| Jikji prepare of a large root times out (`the Jikji prepare run timed out`) | `jikji-prepare-failed` | the refresh still finishes and the other roots are indexed. There is no fixed total wall: the per-root hang guard is `jikji.timeoutMs` (60 s base) plus `jikji.prepareTimeoutMsPerFile` ms per file under the root (default: `jikji.parseTimeout`, 5 s), uncapped, so a huge root is not cut by its size and only a run stuck beyond every file hitting its parse limit is killed. Set `prepareTimeoutMsPerFile: 0` for a flat `timeoutMs` wall. `jikji.parseTimeout` (seconds) is the per-file limit, forwarded to `jikji prepare --parse-timeout`; jikji skips a file that exceeds it. A timeout kills jikji's whole process group |
|
|
857
|
+
| `jikji_find` miss: the `discover` retry step fails (`Jikji discover retry failed: ... (exit status N): <stderr>`) | `jikji-discover-failed` | a miss makes AutoRAG run the `jikji discover ... --after-jikji-retry --retry-proof <proof>` command jikji named (the proof is bound to that root and rewritten query; `--no-background-refresh --no-auto-prepare` keep it from reindexing or rewriting `AGENTS.md`/`CLAUDE.md`/`.cursorrules`). The message carries the CLI exit status and stderr verbatim, real paths included; the failed root is dropped and healthy roots still answer. A `next_commands` line AutoRAG cannot parse as a plain `discover` step is quoted in the same diagnostic and is never executed |
|
|
826
858
|
| Windows file-name search fails during refresh | `everything-index-failed` | read the ES exit code and stderr in the message, then `autorag refresh --method everything --json` |
|
|
827
|
-
|
|
|
859
|
+
| A search stops before answering with `[decision-model-unavailable] ... (missing-credential)` | `decision-model-unavailable` (exit 3) | sign in to Dazzi Models with `/login`, or set `OPENROUTER_API_KEY`; the error's `reason` names what failed (Jev is required) |
|
|
828
860
|
|
|
829
861
|
Native datasource stores stay owned by their CLIs — AutoRAG never rebuilds them. Fix a broken archive with `lazykatok doctor`, `discrawl --json metadata`, `slacrawl --json doctor`, `wacrawl --json doctor`, `telecrawl --json doctor`, `notcrawl doctor`, or `mailcrawl doctor`, then re-run `autorag refresh --method datasources --json`.
|
|
830
862
|
|
package/dist/agent/agent.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type { DatasourceIndexResult, DatasourceSkill } from "../datasource/types
|
|
|
6
6
|
import { type DupeyCliOptions } from "../dupey/index.js";
|
|
7
7
|
import { type Embedder, type ManagedEmbedder } from "../embedding/index.js";
|
|
8
8
|
import { type EverythingClientOptions, type EverythingSearchRequest, type EverythingSearchResult } from "../everything/index.js";
|
|
9
|
+
import { type CloudFileProbe } from "../filesystem/cloud-placeholder.js";
|
|
9
10
|
import { type FSearchClientOptions, type FSearchFailureReason, type FSearchSearchRequest, type FSearchSearchResult } from "../fsearch/index.js";
|
|
10
11
|
import { type JikjiFailureReason, type JikjiOptions, type JikjiPrepareOptions } from "../jikji/index.js";
|
|
11
12
|
import { type LanguageTag } from "../language.js";
|
|
@@ -48,6 +49,22 @@ export interface AutoRAGRefreshLimits {
|
|
|
48
49
|
readonly maxDepth?: number;
|
|
49
50
|
/** Retries for a transient per-file I/O error (EMFILE/EBUSY/EIO/ETIMEDOUT) before the file is skipped. Default 3. */
|
|
50
51
|
readonly transientRetries?: number;
|
|
52
|
+
/**
|
|
53
|
+
* Most cloud-file placeholders (files whose data is not on disk) one refresh may download.
|
|
54
|
+
* Reading a placeholder downloads it, so this bounds disk and network use per refresh; the rest wait
|
|
55
|
+
* for the next one. Default 200.
|
|
56
|
+
*/
|
|
57
|
+
readonly cloudMaxFiles?: number;
|
|
58
|
+
/** Most placeholder bytes one refresh may download; the first file is always allowed. Default 536870912 (512 MiB). */
|
|
59
|
+
readonly cloudMaxBytes?: number;
|
|
60
|
+
/** Release a downloaded cloud file again once it is parsed and indexed. Default true. */
|
|
61
|
+
readonly cloudEvictAfterParse?: boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Retries after a failed release of a downloaded cloud file (`evictUbiquitousItem` can be refused while the
|
|
64
|
+
* provider still holds the file), waiting 500 ms then 1 s then 2 s. After the last retry the file is left on
|
|
65
|
+
* disk and `cloud-evict-failed` is reported. `0` tries once. Default 3.
|
|
66
|
+
*/
|
|
67
|
+
readonly cloudEvictRetries?: number;
|
|
51
68
|
}
|
|
52
69
|
/**
|
|
53
70
|
* Hard caps on retrieval, baseline prefetch, and the candidate lists handed to
|
|
@@ -66,13 +83,18 @@ export interface AutoRAGRetrievalLimits {
|
|
|
66
83
|
readonly minSyncScopedQueryTopK?: number;
|
|
67
84
|
/** Instance scopes listed in one datasource tool description. Default 8. */
|
|
68
85
|
readonly toolDescriptionInstanceScopes?: number;
|
|
69
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Baseline evidence prefetched for the fast answer. Each field's default depends on reranking: with a named rerank
|
|
88
|
+
* model the pool is wide (the reranker picks the best); without one every source contributes its own top 10.
|
|
89
|
+
*/
|
|
70
90
|
readonly prefetch?: {
|
|
71
|
-
/** Jikji find candidate count. Default 30. */
|
|
91
|
+
/** Jikji find candidate count. Default 30 with a reranker, else 10. */
|
|
72
92
|
readonly jikjiTopK?: number;
|
|
73
|
-
/** MinSync retrieve candidate count. Default 100. */
|
|
93
|
+
/** MinSync retrieve candidate count. Default 100 with a reranker, else 10. */
|
|
74
94
|
readonly minSyncTopK?: number;
|
|
75
|
-
/**
|
|
95
|
+
/** Hits requested from each selected datasource. Default `singleDatasourceTopK` (20) with a reranker, else 10. */
|
|
96
|
+
readonly datasourceTopK?: number;
|
|
97
|
+
/** Max Jikji answer paths handed to the verification prompt as starting points. Default 100. */
|
|
76
98
|
readonly jikjiPathLimit?: number;
|
|
77
99
|
/** Max results rendered per baseline section. Default 100. */
|
|
78
100
|
readonly sectionLimit?: number;
|
|
@@ -255,15 +277,20 @@ export interface AutoRAGAgentOptions {
|
|
|
255
277
|
fetch?: WebFetchToolOptions | false;
|
|
256
278
|
}) | false;
|
|
257
279
|
/**
|
|
258
|
-
* Post-merge reranking
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
* merged order is preserved.
|
|
280
|
+
* Post-merge reranking, opt-in: it runs only when a rerank `model` is named. A block without a model is treated as
|
|
281
|
+
* off and reported once as a `rerank-disabled` diagnostic; absent or `false` is off. With reranking off the fast
|
|
282
|
+
* answer's baseline keeps each source's own top 10 (see {@link AutoRAGRetrievalLimits.prefetch}). A configured
|
|
283
|
+
* but unavailable reranker is reported as a diagnostic and the merged order is preserved.
|
|
262
284
|
*/
|
|
263
285
|
rerank?: RerankAgentOptions | false;
|
|
264
286
|
autoRefresh?: AutoRefreshOptions;
|
|
265
287
|
parserOptions?: DefaultParserRegistryOptions;
|
|
266
288
|
dupey?: DupeyCliOptions | false;
|
|
289
|
+
/**
|
|
290
|
+
* Cloud placeholder probe. Omitted, the platform probe is used (macOS and Windows). A programmatic seam for
|
|
291
|
+
* hosts and tests that need to stand in for this machine's cloud and network roots.
|
|
292
|
+
*/
|
|
293
|
+
cloudProbe?: CloudFileProbe;
|
|
267
294
|
/**
|
|
268
295
|
* Filee command line behind `convert_files` and `list_file_conversions`.
|
|
269
296
|
* `false` leaves both tools out. The duplicate-family check on converted
|
|
@@ -271,18 +298,20 @@ export interface AutoRAGAgentOptions {
|
|
|
271
298
|
*/
|
|
272
299
|
filee?: FileeCliOptions | false;
|
|
273
300
|
/**
|
|
274
|
-
* Jev
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
*
|
|
301
|
+
* Jev, the decision model (typed questions in, calibrated probabilities
|
|
302
|
+
* out). It is required: before the fast answer the two-phase search asks
|
|
303
|
+
* it which branch handles the question (local search, file organization,
|
|
304
|
+
* an extra task, or — in a local host — its own configuration) and whether
|
|
305
|
+
* to decompose it; after the fast answer, whether verification is needed.
|
|
306
|
+
* It also judges citations and evidence, drives the `auto` permission
|
|
307
|
+
* gate, and backs the `jev` tool.
|
|
280
308
|
*
|
|
281
|
-
* The
|
|
282
|
-
*
|
|
283
|
-
*
|
|
309
|
+
* The first question of every session probes it and fails with
|
|
310
|
+
* `DecisionModelUnavailableError` (code `decision-model-unavailable`) when
|
|
311
|
+
* it cannot answer. Omitted, the backend is resolved from the environment
|
|
312
|
+
* (the first credential present); `false` is rejected.
|
|
284
313
|
*/
|
|
285
|
-
jev?: JevToolOptions
|
|
314
|
+
jev?: JevToolOptions;
|
|
286
315
|
/**
|
|
287
316
|
* The embedder shared by memory and the datasources. Omitted, it is built from `minSync.embedder`: a
|
|
288
317
|
* configured remote endpoint, else MinSync's `minsync serve` daemon, else MinSync's in-process engine.
|
|
@@ -296,11 +325,10 @@ export interface AutoRAGAgentOptions {
|
|
|
296
325
|
*/
|
|
297
326
|
memoryEmbedder?: Embedder | false;
|
|
298
327
|
/**
|
|
299
|
-
* Agent self-configuration. When set
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
* message.
|
|
328
|
+
* Agent self-configuration. When set, Jev gains a `config` branch for
|
|
329
|
+
* questions about AutoRAG's own settings: the turn skips the fast answer,
|
|
330
|
+
* receives the full `autorag-setup` skill, edits `configPath` with the pi
|
|
331
|
+
* tools, and reports what it changed as its final message.
|
|
304
332
|
*/
|
|
305
333
|
selfConfig?: SelfConfigOptions;
|
|
306
334
|
/**
|
|
@@ -393,7 +421,7 @@ export interface AutoRAGAgentOptions {
|
|
|
393
421
|
export interface RerankAgentOptions {
|
|
394
422
|
/** Provider id. @default "openrouter" */
|
|
395
423
|
provider?: string;
|
|
396
|
-
/** Wire model id.
|
|
424
|
+
/** Wire model id. Required: reranking runs only when a model is named; there is no default. */
|
|
397
425
|
model?: string;
|
|
398
426
|
/** API key. Prefer `apiKeyEnv`; this is a programmatic seam. Not used by the `dazzi` provider. */
|
|
399
427
|
apiKey?: string;
|
|
@@ -497,14 +525,28 @@ export declare class AutoRAGAgent {
|
|
|
497
525
|
private readonly parserOptions;
|
|
498
526
|
private readonly dupeyOptions;
|
|
499
527
|
private readonly fileeOptions;
|
|
500
|
-
/**
|
|
528
|
+
/** Platform cloud probe; `undefined` where AutoRAG has no cloud handling. */
|
|
529
|
+
private readonly cloudProbe;
|
|
530
|
+
/** Resolved once: the machine's non-physical roots, or `[]` when they cannot be listed. */
|
|
531
|
+
private cloudRootsPromise;
|
|
532
|
+
/** pi extension registering the `jev` tool. */
|
|
501
533
|
private readonly jevExtension;
|
|
502
|
-
/**
|
|
534
|
+
/** pi extension releasing the cloud files the agent opened during a run. */
|
|
535
|
+
private readonly cloudRelease;
|
|
536
|
+
/** Jev and its sLLM fallback: the session check picks which one {@link jevJudge} asks. */
|
|
537
|
+
private readonly decisionModels;
|
|
538
|
+
/** Jev judge shared by the `jev` tool, the query router, the judgments, and the permission gate. */
|
|
503
539
|
private readonly jevJudge;
|
|
540
|
+
/**
|
|
541
|
+
* Session whose first question already confirmed that the decision model answers: the bound pi
|
|
542
|
+
* session's id, or {@link UNBOUND_DECISION_SESSION} for runs without a TUI session. Undefined until a
|
|
543
|
+
* check passed; a failed check leaves it unchanged, so the next question checks again.
|
|
544
|
+
*/
|
|
545
|
+
private decisionModelCheckedSession;
|
|
504
546
|
/** Mode and grants of the file-change approval gate; shared by every pi session this agent creates. */
|
|
505
547
|
private readonly permissions;
|
|
506
548
|
private readonly permissionGate;
|
|
507
|
-
/** Self-configuration, enabled only
|
|
549
|
+
/** Self-configuration, enabled only on a local session. */
|
|
508
550
|
private readonly selfConfig;
|
|
509
551
|
/** Override for the packaged organize skill the `organize` branch loads; undefined keeps the packaged one. */
|
|
510
552
|
private readonly organizeSkillPath;
|
|
@@ -532,6 +574,14 @@ export declare class AutoRAGAgent {
|
|
|
532
574
|
private readonly memoryVectorPath;
|
|
533
575
|
/** Every retrieval result of the in-flight run with the query/method that surfaced it. */
|
|
534
576
|
private readonly evidenceOrigins;
|
|
577
|
+
/**
|
|
578
|
+
* Files Jikji found for the current search's queries, in Jikji's order. The baseline prefetch sets it;
|
|
579
|
+
* only the verification prompt uses it, as places to start exploring. The fast answer never sees them:
|
|
580
|
+
* a path says nothing about what a file contains, and the reranker cannot judge one.
|
|
581
|
+
*/
|
|
582
|
+
private jikjiStartingPoints;
|
|
583
|
+
/** Settles once Jikji's find for every query of the current search is done. Never rejects. */
|
|
584
|
+
private jikjiDiscovery;
|
|
535
585
|
/** Search-tool queries by tool call id, read back when the call ends. */
|
|
536
586
|
private readonly searchToolQueries;
|
|
537
587
|
/** Identifies this agent's conversation: everything judged in it is "current" memory. */
|
|
@@ -581,8 +631,8 @@ export declare class AutoRAGAgent {
|
|
|
581
631
|
* After the final answer: map every cited evidence back to the search query
|
|
582
632
|
* and method that found it, ask Jev (one batched call, evidence judged in
|
|
583
633
|
* parallel) whether it really supports the answer sentence it backs, and
|
|
584
|
-
* keep only what clears the threshold.
|
|
585
|
-
*
|
|
634
|
+
* keep only what clears the threshold. A failed judgment stores nothing and
|
|
635
|
+
* says why.
|
|
586
636
|
*/
|
|
587
637
|
private recordJudgedEvidence;
|
|
588
638
|
private resolveSessionModel;
|
|
@@ -615,7 +665,7 @@ export declare class AutoRAGAgent {
|
|
|
615
665
|
private interruptibleBySteer;
|
|
616
666
|
/**
|
|
617
667
|
* Resolve the model's inline `[eN]` citations against this run's evidence and derive the structured results.
|
|
618
|
-
*
|
|
668
|
+
* Every cited chunk is first checked for relevance to the sentence that cites it and to what
|
|
619
669
|
* the user typed in this conversation; a chunk Jev finds unrelated (a table of contents, a footer) is dropped
|
|
620
670
|
* from the citations before the answer is returned or judged for memory.
|
|
621
671
|
*/
|
|
@@ -724,8 +774,8 @@ export declare class AutoRAGAgent {
|
|
|
724
774
|
/**
|
|
725
775
|
* Jev check run after the fast answer: does the answer need correction,
|
|
726
776
|
* clarification, or further research? "No" ends the run with the fast
|
|
727
|
-
* answer as the final answer.
|
|
728
|
-
*
|
|
777
|
+
* answer as the final answer. When the check fails, the run continues into
|
|
778
|
+
* verification.
|
|
729
779
|
*/
|
|
730
780
|
private shouldFollowUp;
|
|
731
781
|
/**
|
|
@@ -735,8 +785,8 @@ export declare class AutoRAGAgent {
|
|
|
735
785
|
* most five search queries with the configured decomposition model
|
|
736
786
|
* (default: the session model). On the local branch, Jev also judges, per
|
|
737
787
|
* registered datasource, whether it must be searched (in parallel with
|
|
738
|
-
* decomposition).
|
|
739
|
-
*
|
|
788
|
+
* decomposition). On any routing failure this is a single local search for
|
|
789
|
+
* the original question with no datasource search.
|
|
740
790
|
* A non-local branch that misses the confidence floor is not a failure: it
|
|
741
791
|
* becomes a full local plan, decomposition and datasource check included.
|
|
742
792
|
*/
|
|
@@ -753,11 +803,20 @@ export declare class AutoRAGAgent {
|
|
|
753
803
|
*/
|
|
754
804
|
private selectSearchDatasources;
|
|
755
805
|
/**
|
|
756
|
-
*
|
|
806
|
+
* Start Jikji's find for every search query without waiting for it. The fast answer never sees Jikji's
|
|
807
|
+
* files (a path says nothing about what a file contains), so nothing in the baseline depends on this; it
|
|
808
|
+
* runs alongside the fast answer and `searchDocuments` waits for it only before the follow-up check, to
|
|
809
|
+
* hand the files to the verification prompt. It stops with the run, not with a steer: a steer makes the
|
|
810
|
+
* run verify anyway, which is when the files are used. A failed find contributes no files.
|
|
811
|
+
*/
|
|
812
|
+
private discoverJikjiFiles;
|
|
813
|
+
/**
|
|
814
|
+
* Baseline local evidence for the fast answer. MinSync, every
|
|
757
815
|
* Jev-selected datasource, and (when Jev selected it) one web search run for
|
|
758
816
|
* every search query in parallel (MinSync itself queues per workspace), the
|
|
759
817
|
* per-query hits are interleaved and deduplicated into one pool, and that
|
|
760
|
-
* pool is reranked against the original question.
|
|
818
|
+
* pool is reranked against the original question. Jikji is started here too
|
|
819
|
+
* (see `discoverJikjiFiles`) but is not awaited and adds nothing to the pool.
|
|
761
820
|
*/
|
|
762
821
|
private prefetchInitialRetrievalContext;
|
|
763
822
|
/**
|
|
@@ -770,7 +829,7 @@ export declare class AutoRAGAgent {
|
|
|
770
829
|
*/
|
|
771
830
|
private prefetchWebSearch;
|
|
772
831
|
/**
|
|
773
|
-
* Rerank the pre-fast-answer baseline pool —
|
|
832
|
+
* Rerank the pre-fast-answer baseline pool — MinSync
|
|
774
833
|
* chunks, Jev-selected datasource chunks, and prefetched web hits — down to
|
|
775
834
|
* the configured `rerank.topN`. With decomposition the pool merges every
|
|
776
835
|
* search query's hits (interleaved), capped at the same per-source sizes as a
|
|
@@ -859,7 +918,7 @@ export declare class AutoRAGAgent {
|
|
|
859
918
|
* against it; the model then verifies with thinking on and ends with the
|
|
860
919
|
* final answer as a plain reply, returning only the delta.
|
|
861
920
|
*/
|
|
862
|
-
buildRefinementPrompt(query: string, options: RetrievalOptions, fastAnswer: string | undefined, fastAnswerDelivered: boolean, steers?: readonly string[]): string;
|
|
921
|
+
buildRefinementPrompt(query: string, options: RetrievalOptions, fastAnswer: string | undefined, fastAnswerDelivered: boolean, steers?: readonly string[], jikjiStartingPoints?: readonly string[]): string;
|
|
863
922
|
buildSearchPrompt(query: string, options: RetrievalOptions, initialRetrievalContext?: string): string;
|
|
864
923
|
refresh(force?: boolean, opts?: AutoRAGRefreshOptions): Promise<AutoRAGRefreshResult>;
|
|
865
924
|
/**
|
|
@@ -893,7 +952,7 @@ export declare class AutoRAGAgent {
|
|
|
893
952
|
*/
|
|
894
953
|
startWatchRefresh(options?: AutoRAGWatchRefreshOptions): AutoRAGWatchRefreshHandle;
|
|
895
954
|
private defaultWatcherFactory;
|
|
896
|
-
syncParsedMirrors(force?: boolean, options?: Pick<ParsedMirrorSyncOptions, "signal" | "onProgress" | "retrySkipped">): Promise<ParsedMirrorSyncResult>;
|
|
955
|
+
syncParsedMirrors(force?: boolean, options?: Pick<ParsedMirrorSyncOptions, "signal" | "onProgress" | "retrySkipped" | "onBatch" | "batchSize">): Promise<ParsedMirrorSyncResult>;
|
|
897
956
|
/**
|
|
898
957
|
* Lightweight stat-only staleness scan used when `refresh` is called with
|
|
899
958
|
* methods that exclude parsed mirrors (e.g. only `datasources` or `jikji`).
|
|
@@ -901,6 +960,8 @@ export declare class AutoRAGAgent {
|
|
|
901
960
|
* so the refresh result and status remain consistent.
|
|
902
961
|
*/
|
|
903
962
|
private scanMirrorStaleness;
|
|
963
|
+
/** The machine's non-physical roots, resolved once; a probe failure reads as no roots, never an error. */
|
|
964
|
+
private cloudRoots;
|
|
904
965
|
private exactDuplicateExclusions;
|
|
905
966
|
syncMinSync(force?: boolean, signal?: AbortSignal, onProgress?: (progress: MinSyncSyncProgress) => void): Promise<MinSyncSyncResult>;
|
|
906
967
|
/** Provider for the Windows-only everything_search tool. */
|
|
@@ -947,6 +1008,15 @@ export declare class AutoRAGAgent {
|
|
|
947
1008
|
readonly first?: boolean;
|
|
948
1009
|
readonly signal?: AbortSignal;
|
|
949
1010
|
}): Promise<JikjiFindProviderResult>;
|
|
1011
|
+
/**
|
|
1012
|
+
* One root's find, followed by jikji's recovery step. When `find` answers
|
|
1013
|
+
* `jikji_retry`, run the `discover` command it named with the proof it
|
|
1014
|
+
* issued for this root and rewritten query (re-running `find` with the proof
|
|
1015
|
+
* does not advance jikji, only `discover` does). One step per find: whatever
|
|
1016
|
+
* discover answers is final for the root, so the protocol cannot loop. A
|
|
1017
|
+
* failed discover drops the root and reports the CLI failure verbatim.
|
|
1018
|
+
*/
|
|
1019
|
+
private findJikjiRoot;
|
|
950
1020
|
/**
|
|
951
1021
|
* Merge per-root answer packs into one. Concatenates answer_paths/candidates
|
|
952
1022
|
* preserving per-root order; dedupes by normalized path. Does NOT cross-root
|
|
@@ -968,7 +1038,8 @@ export declare class AutoRAGAgent {
|
|
|
968
1038
|
* - stopAfterFind: OR
|
|
969
1039
|
* - agentShouldNotRerank: OR
|
|
970
1040
|
* - handoffAction: MOST RESTRICTIVE (direct_use < jikji_retry < raw_fallback_after_retry)
|
|
971
|
-
* - rawFallbackAllowed: handoffAction===raw_fallback_after_retry
|
|
1041
|
+
* - rawFallbackAllowed: handoffAction===raw_fallback_after_retry and every root's raw_fallback_allowed
|
|
1042
|
+
* - maxRawFallbackCommands: MIN over the roots that report one
|
|
972
1043
|
*/
|
|
973
1044
|
private mergePolicy;
|
|
974
1045
|
/**
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ExtensionFactory } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { type CloudFileProbe, type CloudRoot } from "../filesystem/cloud-placeholder.js";
|
|
3
|
+
/** Direct child files one directory candidate contributes; beyond this a directory is left unwatched. */
|
|
4
|
+
export declare const DEFAULT_CLOUD_RELEASE_CHILD_CAP = 500;
|
|
5
|
+
/**
|
|
6
|
+
* Every absolute path in a tool input value: a string that is itself absolute (a `path` argument), and the
|
|
7
|
+
* absolute words of any string that is not (a `bash` command, where `cd "<dir>" && pdftotext "a.pdf" -` names
|
|
8
|
+
* the directory `/…/dir` and no absolute file). Object and array values are walked, so a tool that nests its
|
|
9
|
+
* arguments is covered too.
|
|
10
|
+
*/
|
|
11
|
+
export declare function absolutePathsIn(value: unknown): string[];
|
|
12
|
+
export interface CloudReleaseExtensionOptions {
|
|
13
|
+
/** Platform cloud probe; without one the extension registers nothing (a platform AutoRAG cannot release on). */
|
|
14
|
+
readonly probe?: CloudFileProbe;
|
|
15
|
+
/**
|
|
16
|
+
* The machine's non-physical roots, resolved lazily and cached by the caller. Without one, or when it
|
|
17
|
+
* resolves to no releasable root (Linux, or a machine with no cloud folders), the extension does nothing.
|
|
18
|
+
*/
|
|
19
|
+
readonly cloudRoots?: () => Promise<readonly CloudRoot[]>;
|
|
20
|
+
/** Direct child files one directory candidate contributes. Default {@link DEFAULT_CLOUD_RELEASE_CHILD_CAP}. */
|
|
21
|
+
readonly childCap?: number;
|
|
22
|
+
/** Retries after the first failed release; `0` tries once. Default {@link DEFAULT_CLOUD_EVICT_RETRIES}. */
|
|
23
|
+
readonly evictRetries?: number;
|
|
24
|
+
/** First retry wait in ms, doubled each retry. Default {@link DEFAULT_CLOUD_EVICT_BACKOFF_MS}. */
|
|
25
|
+
readonly evictBackoffMs?: number;
|
|
26
|
+
/** Wait implementation; tests inject one that records instead of sleeping. */
|
|
27
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
28
|
+
/** One line per file that could not be released; defaults to stderr. Never thrown, never shown in the UI. */
|
|
29
|
+
readonly log?: (message: string) => void;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* pi extension that releases cloud files the agent itself opened during an answer.
|
|
33
|
+
*
|
|
34
|
+
* A refresh releases the placeholders it downloads, but the agent's own tools (`bash`, `read`, `grep`, …) open
|
|
35
|
+
* originals too, and nothing released those. This extension watches the files a run named or was shown: before
|
|
36
|
+
* each call it stats the candidates a tool input names (an absolute `path`, the absolute words of a `bash`
|
|
37
|
+
* command, the direct children of a directory candidate) and remembers the ones that are placeholders now;
|
|
38
|
+
* after each result it does the same for the absolute paths a retrieval result printed. At `agent_end` it
|
|
39
|
+
* re-stats what it remembered and releases each file that is now on disk. A file that was already local when
|
|
40
|
+
* first seen is never watched, so only what this run hydrated is released. Without a probe or without a
|
|
41
|
+
* releasable cloud root the extension does nothing; a failed release is logged once and never thrown.
|
|
42
|
+
*/
|
|
43
|
+
export declare function createCloudReleaseExtension(options: CloudReleaseExtensionOptions): ExtensionFactory;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { ClassifierApi, ClassifierContext, ClassifierModel, ClassifierResult, ModelsClassifierOptions } from "@earendil-works/pi-ai";
|
|
2
|
+
import { type BackendRequest, type BackendResponse, type JevBackend } from "jev-use";
|
|
3
|
+
/**
|
|
4
|
+
* Decision models come from pi's model catalog: pi-ai ships every provider's typed-question
|
|
5
|
+
* ("classifier") models next to its chat models, with the transport for each wire dialect
|
|
6
|
+
* (System One on TypeSafe, OpenRouter, Vercel AI Gateway, Cloudflare Workers AI, and OpenCode;
|
|
7
|
+
* OpenAI's Decisions API for GPT-6 Luna), and pi's runtime adds the credentials in `auth.json` and
|
|
8
|
+
* the provider `baseUrl` overrides in `models.json` (a proxy or gateway speaking the same dialect).
|
|
9
|
+
* {@link PiDecisionBackend} puts one of those models behind jev-use's backend contract, so the
|
|
10
|
+
* jev-use engine (screening, escalation) and every AutoRAG judgment stay unchanged.
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* The decision model asked on each provider when the config names none. OpenRouter stays on
|
|
14
|
+
* `typesafe/jev-1.13`, the model AutoRAG's routing and permission thresholds were calibrated on.
|
|
15
|
+
*/
|
|
16
|
+
export declare const DECISION_PROVIDER_DEFAULT_MODELS: Readonly<Record<string, string>>;
|
|
17
|
+
/** The pi providers tried, in this order, when the caller names no decision backend. */
|
|
18
|
+
export declare const DECISION_PROVIDER_ORDER: readonly string[];
|
|
19
|
+
/** What {@link PiDecisionBackend} reads from pi's model runtime. pi's `ModelRuntime` is one. */
|
|
20
|
+
export interface DecisionModelRuntime {
|
|
21
|
+
getModelsOfType(type: "classifier", provider?: string): readonly ClassifierModel<ClassifierApi>[];
|
|
22
|
+
classify(model: ClassifierModel<ClassifierApi>, context: ClassifierContext, options?: ModelsClassifierOptions): Promise<ClassifierResult>;
|
|
23
|
+
}
|
|
24
|
+
export interface PiDecisionBackendOptions {
|
|
25
|
+
/** pi provider id with decision models: `openrouter`, `openai`, `typesafe`, `vercel-ai-gateway`, `cloudflare-workers-ai`, or `opencode`. */
|
|
26
|
+
readonly provider: string;
|
|
27
|
+
/** Decision model id within the provider. Default: {@link DECISION_PROVIDER_DEFAULT_MODELS}, else the provider's first. */
|
|
28
|
+
readonly model?: string;
|
|
29
|
+
/** pi's runtime; built from `agentDir` on the first call when omitted. */
|
|
30
|
+
readonly runtime?: DecisionModelRuntime;
|
|
31
|
+
/** Pi agent home whose `auth.json` and `models.json` the runtime reads. Default: Pi's agent dir. */
|
|
32
|
+
readonly agentDir?: string;
|
|
33
|
+
/** Environment the provider credential is read from, ahead of `process.env`. */
|
|
34
|
+
readonly env?: Record<string, string | undefined>;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* One decision model from pi's catalog, as a jev-use backend. Missing credentials, an unknown model,
|
|
38
|
+
* and provider errors surface as `BackendError`s with the provider's own message, which the jev-use
|
|
39
|
+
* engine turns into `unreachable` escalations and the session check into a typed reason.
|
|
40
|
+
*/
|
|
41
|
+
export declare class PiDecisionBackend implements JevBackend {
|
|
42
|
+
readonly name: string;
|
|
43
|
+
private readonly modelId;
|
|
44
|
+
private readonly runtime;
|
|
45
|
+
private readonly agentDir;
|
|
46
|
+
private readonly env;
|
|
47
|
+
constructor(options: PiDecisionBackendOptions);
|
|
48
|
+
judge(request: BackendRequest): Promise<BackendResponse>;
|
|
49
|
+
}
|