@autorag/librarian 2.10.6 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/NOTICE +14 -0
  2. package/README.md +52 -10
  3. package/dist/agent/agent.d.ts +70 -2
  4. package/dist/agent/answer-citations.d.ts +34 -2
  5. package/dist/agent/answer-guidelines.d.ts +24 -3
  6. package/dist/agent/ask-user.d.ts +8 -0
  7. package/dist/agent/automation-tool.d.ts +45 -0
  8. package/dist/agent/citation-links.d.ts +23 -0
  9. package/dist/agent/citation-numbers.d.ts +68 -0
  10. package/dist/agent/citation-relevance.d.ts +37 -0
  11. package/dist/agent/conversation-queries.d.ts +31 -0
  12. package/dist/agent/convert-tool.d.ts +28 -0
  13. package/dist/agent/evidence-persistence.d.ts +17 -11
  14. package/dist/agent/index.d.ts +1 -0
  15. package/dist/agent/intent-tracing.d.ts +33 -0
  16. package/dist/agent/mcp-client.d.ts +27 -0
  17. package/dist/agent/permission-gate.d.ts +78 -0
  18. package/dist/agent/permission-judgment.d.ts +72 -0
  19. package/dist/agent/permission-paths.d.ts +23 -0
  20. package/dist/agent/pi-session.d.ts +8 -0
  21. package/dist/agent/query-routing.d.ts +14 -17
  22. package/dist/agent/search-documents.d.ts +1 -1
  23. package/dist/agent/search-progress.d.ts +96 -0
  24. package/dist/agent/self-config.d.ts +2 -0
  25. package/dist/agent/user-requests.d.ts +52 -0
  26. package/dist/agent/welcome-header.d.ts +35 -0
  27. package/dist/automation/config.d.ts +48 -0
  28. package/dist/automation/daemon.d.ts +31 -0
  29. package/dist/automation/dispatch.d.ts +13 -0
  30. package/dist/automation/run-agent.d.ts +22 -0
  31. package/dist/automation/run.d.ts +32 -0
  32. package/dist/automation/service.d.ts +61 -0
  33. package/dist/cli/commands/automate.d.ts +2 -0
  34. package/dist/cli/commands/setup.d.ts +2 -0
  35. package/dist/cli/config.d.ts +21 -0
  36. package/dist/cli/index.d.ts +1 -1
  37. package/dist/cli/index.js +26740 -21094
  38. package/dist/cli/open-target.d.ts +9 -0
  39. package/dist/cli/setup.d.ts +21 -0
  40. package/dist/convert/cli.d.ts +77 -0
  41. package/dist/convert/index.d.ts +4 -0
  42. package/dist/convert/installer.d.ts +84 -0
  43. package/dist/convert/matrix.d.ts +27 -0
  44. package/dist/convert/relate.d.ts +30 -0
  45. package/dist/core.js +32095 -27288
  46. package/dist/datasource/connector-timeouts.d.ts +6 -2
  47. package/dist/datasource/crawler-types.d.ts +1 -1
  48. package/dist/datasource/errors.d.ts +1 -1
  49. package/dist/datasource/index.d.ts +2 -1
  50. package/dist/datasource/skills/factory.d.ts +3 -0
  51. package/dist/datasource/skills/hister/client.d.ts +83 -0
  52. package/dist/datasource/skills/hister/embedding.d.ts +64 -0
  53. package/dist/datasource/skills/hister/index.d.ts +2 -0
  54. package/dist/datasource/skills/hister/setup.d.ts +87 -0
  55. package/dist/datasource/skills/hister/skill.d.ts +63 -0
  56. package/dist/datasource/skills/lazykatok/types.d.ts +2 -2
  57. package/dist/datasource/skills/mailcrawl/types.d.ts +1 -1
  58. package/dist/datasource/skills/obsidian/index.d.ts +4 -6
  59. package/dist/datasource/skills/obsidian/methods.d.ts +13 -17
  60. package/dist/datasource/skills/obsidian/paths.d.ts +24 -24
  61. package/dist/datasource/skills/obsidian/skill.d.ts +10 -15
  62. package/dist/datasource/skills/obsidian/staging.d.ts +50 -0
  63. package/dist/datasource/skills/obsidian/vault-index.d.ts +44 -0
  64. package/dist/dupey/cli.d.ts +18 -0
  65. package/dist/index.d.ts +1 -0
  66. package/dist/index.js +32263 -27429
  67. package/dist/mcp/index.js +32149 -27342
  68. package/dist/minsync/installer.d.ts +1 -1
  69. package/dist/minsync/workspace-lock.d.ts +7 -0
  70. package/dist/process/terminate-tree.d.ts +4 -5
  71. package/dist/profiles/agent-factory.d.ts +60 -0
  72. package/dist/profiles/automation-profiles.d.ts +25 -0
  73. package/dist/profiles/loop-guard.d.ts +23 -0
  74. package/dist/profiles/retrieval-tools.d.ts +24 -0
  75. package/dist/profiles/session-model.d.ts +69 -0
  76. package/dist/profiles.d.ts +13 -0
  77. package/dist/profiles.js +34739 -0
  78. package/dist/retrieval/rerank.d.ts +6 -0
  79. package/licenses/hister-AGPL-3.0-notice.txt +47 -0
  80. package/package.json +9 -3
  81. package/skills/autorag/SKILL.md +79 -5
  82. package/skills/autorag-doctor/SKILL.md +5 -4
  83. package/skills/autorag-lite-setup/SKILL.md +23 -5
  84. package/skills/autorag-setup/SKILL.md +68 -7
  85. package/dist/datasource/skills/obsidian/client.d.ts +0 -15
  86. package/dist/datasource/skills/obsidian/types.d.ts +0 -92
@@ -33,6 +33,12 @@ export declare const DEFAULT_DAZZI_RERANK_MODEL = "qwen/qwen3-reranker-8b";
33
33
  export declare const DEFAULT_RERANK_TIMEOUT_MS = 60000;
34
34
  /** Default number of merged chunks kept after reranking. */
35
35
  export declare const DEFAULT_RERANK_TOP_N = 25;
36
+ /**
37
+ * Longest document, in UTF-16 units, sent to a reranker. The hosted Dazzi
38
+ * reranker rejects a document of 131,072 characters or more with HTTP 400 and
39
+ * fails the whole request; 131,000 stays under that with a small margin.
40
+ */
41
+ export declare const RERANK_MAX_DOCUMENT_CHARS = 131000;
36
42
  /** Rerank providers {@link createReranker} can build. */
37
43
  export declare const SUPPORTED_RERANK_PROVIDERS: readonly string[];
38
44
  /** Default wire model for a rerank provider. */
@@ -0,0 +1,47 @@
1
+ Hister — license notice for AutoRAG's `hister` datasource
2
+ ==========================================================
3
+
4
+ Program: Hister ("Your own search engine")
5
+ Source: https://github.com/asciimoo/hister
6
+ Copyright: asciimoo and Hister contributors (see the repository history)
7
+ License: GNU Affero General Public License, version 3 or (at your option)
8
+ any later version (AGPL-3.0-or-later)
9
+ License text: https://www.gnu.org/licenses/agpl-3.0.txt
10
+ (also the LICENSE file in the Hister repository)
11
+
12
+ How AutoRAG relates to Hister
13
+ -----------------------------
14
+ * AutoRAG does NOT contain, bundle, link, vendor, or redistribute any Hister
15
+ source code or binary. Nothing from the Hister repository is copied into
16
+ this package.
17
+ * Users install Hister themselves, unmodified, from its own releases
18
+ (https://github.com/asciimoo/hister/releases, `brew install hister`, Docker,
19
+ or Nix), and run it as a separate process (`hister listen`).
20
+ * AutoRAG talks to that process only over HTTP, using Hister's public JSON API
21
+ (`GET /search`, `GET /api/config`), the same API Hister's own `hister search`
22
+ command uses. Hister owns its index, its browser extension, its credentials,
23
+ and its optional embedding calls.
24
+ * AutoRAG does not modify Hister, and it never hides that Hister is the backend:
25
+ the datasource manifest shown to the agent names Hister, its license, and its
26
+ source URL.
27
+ * The `hister` datasource only reads web-page documents (`type:web`). It never
28
+ enables or uses Hister's local-directory indexing.
29
+
30
+ Combining separate programs that communicate over a network is "mere
31
+ aggregation" in the sense of the GNU licenses FAQ
32
+ (https://www.gnu.org/licenses/gpl-faq.html#MereAggregation); AutoRAG therefore
33
+ stays under its own license (Apache-2.0).
34
+
35
+ If you run Hister as a network service for other people, AGPL-3.0 section 13
36
+ applies to YOU as the operator of that Hister instance: you must offer those
37
+ users the Corresponding Source of the Hister version you run (and of any
38
+ modifications you made). Running an unmodified upstream release satisfies this
39
+ by pointing users at https://github.com/asciimoo/hister.
40
+
41
+ Obtaining the Hister source for a release you run
42
+ -------------------------------------------------
43
+ git clone https://github.com/asciimoo/hister
44
+ git checkout <the version printed by `hister --version`>
45
+
46
+ This file is a notice, not the license. The authoritative license terms are in
47
+ the Hister repository and at the AGPL URL above.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@autorag/librarian",
3
- "version": "2.10.6",
4
- "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
3
+ "version": "2.12.0",
4
+ "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
7
7
  "type": "git",
@@ -18,6 +18,10 @@
18
18
  "./core": {
19
19
  "types": "./dist/core.d.ts",
20
20
  "default": "./dist/core.js"
21
+ },
22
+ "./profiles": {
23
+ "types": "./dist/profiles.d.ts",
24
+ "default": "./dist/profiles.js"
21
25
  }
22
26
  },
23
27
  "bin": {
@@ -36,7 +40,7 @@
36
40
  "access": "public"
37
41
  },
38
42
  "scripts": {
39
- "build": "bun build src/index.ts src/core.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
43
+ "build": "bun build src/index.ts src/core.ts src/profiles.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
40
44
  "prepublishOnly": "bun run typecheck && bun run build",
41
45
  "prepare": "bun run build",
42
46
  "test": "node scripts/run-test-files.mjs",
@@ -56,6 +60,7 @@
56
60
  "@earendil-works/pi-tui": "1.2.0",
57
61
  "@llblab/pi-telegram": "0.57.2",
58
62
  "@modelcontextprotocol/server": "^2.3.1",
63
+ "@parcel/watcher": "2.6.0",
59
64
  "chardet": "^2.2.0",
60
65
  "discord.js": "14.27.0",
61
66
  "fast-xml-parser": "^5.11.2",
@@ -65,6 +70,7 @@
65
70
  "kordoc": "^4.21.30",
66
71
  "mailparser": "^3.9.38",
67
72
  "pdfjs-dist": "^6.4.299",
73
+ "pi-ask-user": "0.17.0",
68
74
  "pi-discord-remote": "0.2.4",
69
75
  "puppeteer-core": "^25.13.0",
70
76
  "slides-grab": "^1.5.2",
@@ -15,10 +15,13 @@ search, calls MinSync, Jikji, datasource, and filesystem tools, reads
15
15
  sources, judges evidence, and curates the final answer. There is no subagent or
16
16
  separate model role.
17
17
 
18
- AutoRAG reads source documents and writes indexes only under the configured
18
+ Indexing and refresh read source documents and write only under the configured
19
19
  workspace `.autorag/` directory and Jikji's central store (`<OS data dir>/jikji/`,
20
- plus a `.jikji_agent_map.md` guide in each indexed root). Never
21
- move, rename, edit, or delete source files.
20
+ plus a `.jikji_agent_map.md` guide in each indexed root). The librarian may
21
+ organize, create, or modify the user's own files in local sessions when the
22
+ user asks; it never deletes permanently (OS trash only) and lists every change
23
+ as old path → new path. Do not move, rename, edit, or delete source files on your own
24
+ initiative while searching or diagnosing.
22
25
 
23
26
  ## Preflight
24
27
 
@@ -42,6 +45,34 @@ near/contains families require human review. Exact duplicate exclusion during
42
45
  refresh is enabled by default and can be disabled with
43
46
  `"excludeExactDuplicates": false` in `config.json`.
44
47
 
48
+ ### Converting files
49
+
50
+ The `convert_files` and `list_file_conversions` Agent tools run the
51
+ [Filee](https://github.com/KnifeLemon/Filee) command line to convert local
52
+ files offline: documents, spreadsheets, slides, HWP/HWPX, images, e-books,
53
+ archives, audio, video, data, and subtitles. They work on any path the user
54
+ names, not only the configured roots. This is the one exception to the
55
+ "never write source files" rule above: a conversion writes a new file and
56
+ never touches its source. An existing output name gets a number added unless
57
+ the user asks to replace it, in which case pass `overwrite: true`. Never
58
+ delete the source unless the user asked.
59
+
60
+ Each converted file is checked with `dupey compare` against its source, using
61
+ dupey's own thresholds, so the report says whether a later scan will group
62
+ them: `exact` (same canonical text, for example DOCX to HWPX), `near` (same
63
+ document in another format, for example Markdown to TXT), `different` (text
64
+ differs, typically layout formats such as PDF), `unsupported` (dupey cannot
65
+ read the format, for example HTML or images), or `unavailable` (dupey not
66
+ installed). The command line is `filee.binaryPath` if set, else a `filee`/`filee-cli`
67
+ on `PATH`, else the pinned [filee-cli](https://github.com/NomaDamas/filee-cli)
68
+ release that `autorag setup` (or the first conversion) downloads into
69
+ `<AUTORAG_HOME>/tools/filee-cli/`; `"filee": { "autoInstall": false }` forbids that
70
+ download. The system prompt carries the conversion map (which format family
71
+ converts to which, and the two-step chain when no direct route exists), so plan
72
+ from it instead of probing. `"filee": { "enabled": false }` in `config.json` removes both
73
+ tools; `filee.timeoutMs` (default 30 minutes) and `filee.dataDir` (Filee's
74
+ own settings and engines, `FILEE_DATA_DIR`) tune them.
75
+
45
76
  ### Excluding local files from the index
46
77
 
47
78
  `"excludePaths"` in `config.json` lists files or folders (absolute, or relative
@@ -80,7 +111,10 @@ autorag search "what were the key findings in the Q3 report" --top-k 5 --json --
80
111
  only `answer` plus `results`. Every
81
112
  bracketed `[n]` citation in `answer` resolves to a `results[].number` of the
82
113
  same response; an unmatched citation is removed and reported as a
83
- `citation-without-result` diagnostic.
114
+ `citation-without-result` diagnostic. Within one conversation (an interactive
115
+ session, including a resumed one), a source keeps the `number` it was first
116
+ shown with and a new source gets the next unused number, so `results[].number`
117
+ can skip numbers and need not start at 1 on a later turn.
84
118
 
85
119
  To inspect the exact persisted evidence behind numbered results, use:
86
120
 
@@ -113,7 +147,13 @@ default, OpenRouter), Jev first routes the question:
113
147
  datasources) is handled by the agent itself: it loads the setup skill, edits
114
148
  the active config, verifies it, and reports what changed. The running agent
115
149
  keeps its startup model; changes apply to the next `autorag` launch.
116
- - Private-data questions use local search; public current facts use web search.
150
+ - A request to act on files or folders (move, bulk rename, rename by content,
151
+ convert, sort) is carried out in one tool turn with no retrieval and no
152
+ verification, and reports every change as old path → new path.
153
+ - Anything else that needs no retrieval (commit, push, open a PR, run a build)
154
+ is carried out in one tool turn the same way.
155
+ - Private-data questions use local search; the agent decides on web search
156
+ after the fast answer.
117
157
  - A multi-part question is split into up to five parallel search queries.
118
158
  - On local search, Jev also picks which registered datasources (Slack,
119
159
  Discord, KakaoTalk, email, ...) to search before the fast answer, from each
@@ -142,8 +182,42 @@ autorag index rebuild --yes --json
142
182
  autorag index reset --method parsed --yes --json
143
183
  autorag memory inspect --json
144
184
  autorag tui
185
+ autorag automate status --json
186
+ autorag automate install --json
187
+ autorag automate uninstall --json
188
+ autorag automate run invoice --event create:/path/to/file.pdf --json
189
+ autorag automate daemon
190
+ ```
191
+
192
+ File-triggered automations live in `~/.autorag/automations.json` (`automations`
193
+ array of `{ id, paths, events, include, exclude, run | agent, appendFiles, debounceMs, timeoutMs }`).
194
+ Edit `automations.json` directly to add or change rules; matched `[{ type, path }]`
195
+ events are piped on stdin to `run`. Use `autorag automate install` to register
196
+ the background daemon with launchd, `systemd --user`, or Windows Task Scheduler.
197
+
198
+ A rule runs exactly one of `run` (a command) or `agent` (an AI agent started in
199
+ the daemon on each batch of changes):
200
+
201
+ ```json
202
+ { "id": "inbox-triage", "paths": ["~/Downloads"], "include": ["**/*.pdf"],
203
+ "agent": { "profile": "file-triage", "task": "Summarize each new PDF.",
204
+ "retrieval": { "searchAll": true, "datasources": ["kakao"] } } }
145
205
  ```
146
206
 
207
+ `agent.profile` names a profile defined in code: `file-triage` (reads, lists and
208
+ searches files; changes nothing) or `file-organizer` (also `bash`, `edit`,
209
+ `write`). A rule never carries a prompt or a tool list, so the rules file cannot
210
+ grant an agent more than its profile has. `retrieval` optionally adds
211
+ `search_all_documents` and one `search_datasource_<id>` per listed datasource.
212
+ Retrieval memory is never connected to an automation agent.
213
+
214
+ Every agent run gets a loop guard appended to its system prompt, naming the
215
+ rule's folders, file types and events, and forbidding any change that would
216
+ fire the rule again, editing the rules file, and running `autorag automate`. At
217
+ most one agent run is in flight across all rules, and an agent rule is
218
+ disabled after 6 runs within 60 seconds (command rules: 20). The run's final
219
+ reply is the `stdout` of its log entry.
220
+
147
221
  Prefer a full refresh so parsed mirrors, MinSync, Jikji, configured
148
222
  datasources, and (on Windows) the bundled Everything file-name index stay
149
223
  aligned. `--method` accepts
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: autorag-doctor
3
- description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
3
+ description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
4
4
  license: Apache-2.0
5
5
  ---
6
6
 
@@ -15,9 +15,10 @@ Always finish with the status table in [Report](#report).
15
15
 
16
16
  ## Safety
17
17
 
18
- - Never delete, move, or edit source documents.
18
+ - Diagnosis and repair never delete, move, or edit source documents; a doctor
19
+ run changes only AutoRAG-owned state.
19
20
  - Never delete a datasource's **native** store (`~/Library/Application Support/katok`, `~/.discrawl`,
20
- `~/.mailcrawl`, `.qmd`, Telegram/WhatsApp/Notion archives). AutoRAG only
21
+ `~/.mailcrawl`, Telegram/WhatsApp/Notion archives). AutoRAG only
21
22
  reads them; rebuilding them is the owning CLI's job.
22
23
  - Only AutoRAG-owned state under `AUTORAG_HOME` and the workspace `.autorag`
23
24
  directory may be reset.
@@ -69,7 +70,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
69
70
  | WhatsApp | `wacrawl --json doctor` | `wacrawl import` |
70
71
  | Telegram | `telecrawl --json doctor`, `telecrawl --json status` | `telecrawl import` |
71
72
  | Notion | `notcrawl doctor`, `notcrawl status` | `notcrawl sync --source desktop` |
72
- | Obsidian / notes | `qmd status` | `qmd update && qmd embed` |
73
+ | Obsidian / notes | `autorag status` (AutoRAG-owned index) | `autorag refresh --method datasources`; if corrupt, delete `.autorag/datasources/obsidian/<instance>` first |
73
74
  | Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
74
75
  | Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
75
76
  | Spotlight (macOS) | `mdutil -s /` | indexed by the OS; no AutoRAG sync |
@@ -17,9 +17,9 @@ Use `autorag-setup` instead when the model-backed librarian must be configured.
17
17
  `memoryPath`, `minSync`, `jikji`, `datasources`.
18
18
  - Never print, copy, migrate, compare, or persist credential values. Store only
19
19
  environment-variable names such as `tokenEnv` or `apiKeyEnv`.
20
- - Never move, rename, edit, or delete source documents. Lite commands write
21
- indexes only under the configured workspace `.autorag/` directory and
22
- Jikji's central store (`<OS data dir>/jikji/`).
20
+ - Setup and Lite commands never move, rename, edit, or delete source documents.
21
+ They write indexes only under the configured workspace `.autorag/` directory
22
+ and Jikji's central store (`<OS data dir>/jikji/`).
23
23
  - Do not index system trees, app bundles, caches, credential stores,
24
24
  `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
25
25
  `.jikji`.
@@ -113,7 +113,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
113
113
 
114
114
  1. Probe every datasource for setup feasibility before asking the user
115
115
  anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
116
- `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`) and its
116
+ `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`) and its
117
117
  local store or archive is present. CLI-backed datasources own their own
118
118
  archive, index, and authentication, so environment credentials (such as bot
119
119
  tokens) are never required or checked for them. Non-CLI connectors
@@ -137,7 +137,7 @@ Datasource skills belong in trusted config.
137
137
 
138
138
  Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
139
139
  `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
140
- `mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`) or connection
140
+ `mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, `hister`) or connection
141
141
  aliases with `"type": "<template>"`. Unknown names are skipped with an
142
142
  `unknown-datasource-skill` warning; they do not fail config resolution.
143
143
  `scope` narrows a query to a sub-path as ordinary filtering. Tags are
@@ -146,6 +146,24 @@ select configured connections before retrieval; discover their IDs with
146
146
  `autorag.datasources.list`. Store only env-var names such as `tokenEnv` or
147
147
  `apiKeyEnv`, never credential values.
148
148
 
149
+ ### Browser history (Hister) and the shared embedding model
150
+
151
+ `hister` needs a separately installed, always-running Hister server (AGPL-3.0, never
152
+ bundled by AutoRAG). Run `autorag setup --format json` and read the `hister` object
153
+ (`installed`, `running`, `embedding`); each problem carries the exact command to run:
154
+ `brew install hister`, `brew services start hister`, then install the Hister browser
155
+ extension in the user's browser.
156
+
157
+ Hister embeds with its own `semantic_search` config, so it must use the same model as
158
+ AutoRAG (the `minsync serve` daemon, `Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions) or
159
+ its vectors stop lining up without any error. `autorag setup --sync-hister` copies the
160
+ shared settings into Hister's `config.yml` (backup kept), restarts the service, waits
161
+ for it, and runs `hister reindex` when needed; it is idempotent, and it refuses a
162
+ remote or in-process backend. Re-run it after restarting the `minsync serve` daemon
163
+ (its port changes). Verify with `hister doctor`: `server.index.embeddings` must be `ok`.
164
+ Install a skip list in Hister's `rules.json` before importing past history, because the
165
+ importer re-downloads URLs without the browser's cookies (docs/datasource-skills.md).
166
+
149
167
  ## Build and refresh indexes
150
168
 
151
169
  After connecting, call MCP `autorag.refresh` with `{}` for an incremental run.
@@ -21,7 +21,8 @@ librarian, not the Lite MCP server.
21
21
  - Never print, copy, migrate, compare, or persist credential values. Store only
22
22
  environment-variable names such as `apiKeyEnv`.
23
23
  - Do not scan the whole filesystem or home directory without explicit approval.
24
- - Never move, rename, edit, or delete source documents.
24
+ - Setup, indexing, and refresh never move, rename, edit, or delete source
25
+ documents. The librarian itself may organize files when the user asks it to.
25
26
  - Do not index system trees, app bundles, caches, credential stores,
26
27
  `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
27
28
  `.jikji`.
@@ -200,7 +201,7 @@ resolves in order:
200
201
 
201
202
  1. A configured remote endpoint (`minSync.embedder`), used for everything; no
202
203
  daemon is started and no local model is loaded.
203
- 2. A running `minsync serve` daemon (loopback only; MinSync 0.4.11+).
204
+ 2. A running `minsync serve` daemon (loopback only; MinSync 0.4.12+).
204
205
  AutoRAG never starts it: run `minsync serve --idle-timeout 0` yourself, as
205
206
  a launchd/systemd service (see `docs/minsync-setup.md`), or press
206
207
  Ctrl+Shift+E in `autorag tui`. `autorag setup` only reports whether it is
@@ -238,9 +239,11 @@ fast and multi-part questions thorough.
238
239
  - **Jev** (`jev`, default `{ "backend": "dazzi" }` when signed in to Dazzi
239
240
  Models, else `{ "backend": "openrouter" }`; model `jev-1.13` on Dazzi,
240
241
  `typesafe/jev-1.13` on OpenRouter) runs before the fast answer. It routes each question to
241
- local search, web search, a direct answer (general knowledge or small
242
- talk skips retrieval entirely), or the `config` branch, and decides whether
243
- to decompose it. On
242
+ local search, a direct answer (general knowledge or small
243
+ talk skips retrieval entirely), `organize` (moving, renaming, converting, or
244
+ sorting files and folders), `extra` (any other action that needs no
245
+ retrieval, such as committing code), or the `config` branch, and decides
246
+ whether to decompose it. On
244
247
  local search it also decides, per registered datasource, whether to search
245
248
  it before the fast answer, using each datasource's `description` and where
246
249
  similar past questions were answered (retrieval memory). When setting up a
@@ -361,7 +364,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
361
364
 
362
365
  1. Probe every datasource for setup feasibility before asking the user
363
366
  anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
364
- `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`, `lark-cli`) and its
367
+ `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`, `lark-cli`) and its
365
368
  local store or archive is present. CLI-backed datasources own their own
366
369
  archive, index, and authentication, so environment credentials (such as bot
367
370
  tokens) are never required or checked for them. Non-CLI connectors
@@ -384,7 +387,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
384
387
  Datasource skills belong in trusted config. Builtin
385
388
  template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
386
389
  `clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
387
- `mailcrawl`, `obsidian`, `rss`, `spotlight`, and `lark`. Config keys may be connection
390
+ `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, and `hister`. Config keys may be connection
388
391
  aliases with `"type": "<template>"`. Unknown names are skipped with an
389
392
  `unknown-datasource-skill` warning; they do not fail config resolution.
390
393
  `scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.
@@ -418,6 +421,62 @@ the next `index`. kakao needs `lazykatok` 0.3.3 or newer for the same reason.
418
421
  Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
419
422
  datasource is removed.
420
423
 
424
+ ### Browser history (Hister) and the shared embedding model
425
+
426
+ `hister` is the one datasource whose backing program AutoRAG does not install,
427
+ start, or bundle (it is AGPL-3.0-or-later and stays a separate program). Setting
428
+ it up is an agent task with these steps. Run each command; do not just describe it.
429
+
430
+ 1. **Probe first.** Add the datasource entry, then run `autorag setup --format json`
431
+ and read the `hister` object: `installed`, `running`, and `embedding`. Every
432
+ problem has a `remediation` string with the exact next command.
433
+ 2. **Install Hister** when `installed` is false: `brew install hister` (other
434
+ platforms: the release binaries at https://github.com/asciimoo/hister/releases).
435
+ 3. **Run it as a login item** when `running` is false: `brew services start hister`.
436
+ Hister must stay running: its browser extension only captures while it is up,
437
+ and a page visited while it is down is not re-sent. AutoRAG never starts it.
438
+ 4. **Install the browser extension** (Chrome Web Store "Hister", or Firefox
439
+ Add-ons). It must be done in the user's browser; tell the user if you cannot.
440
+ 5. **Share the embedding model** (below).
441
+ 6. **Before importing past history, install a skip list** in Hister's `rules.json`
442
+ (login, mail, documents, social and console domains, private hosts, token-bearing
443
+ URLs). `hister import browser history` re-downloads each URL *without* the
444
+ browser's cookies. See docs/datasource-skills.md, "Browser history via Hister".
445
+ 7. Add `"browser": { "type": "hister", "connector": { "semantic": true } }` under
446
+ `datasources`, then `autorag refresh` (only probes Hister) and
447
+ `autorag lite retrieve "<a question>" --scope '/browser/**' --json`.
448
+
449
+ **Shared embedding model.** Hister keeps its own vector store and embeds with
450
+ whatever its `config.yml` `semantic_search` block says. Pointing it at a different
451
+ model does not fail: vectors simply stop lining up and results get worse. So Hister
452
+ must use the same model as AutoRAG, which is the `minsync serve` daemon (model
453
+ `Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, with Qwen3's query instruction).
454
+
455
+ ```bash
456
+ autorag setup --format json # reports hister.embedding.differs: the keys that disagree
457
+ autorag setup --sync-hister # writes them, restarts Hister, re-embeds pages if needed
458
+ ```
459
+
460
+ - `--sync-hister` rewrites only `enable`, `embedding_endpoint`, `embedding_model`,
461
+ `dimensions`, `query_prefix`, `document_prefix` in Hister's config (a
462
+ `config.yml.bak.<time>` is kept next to it), then runs `brew services restart hister`,
463
+ waits for it to answer, and runs `hister reindex` when a change makes stored
464
+ vectors stale. Without the flag setup never writes Hister's config.
465
+ - The daemon port is chosen when `minsync serve` starts. After restarting the daemon,
466
+ run `autorag setup --sync-hister` again. It is idempotent: when nothing differs
467
+ it changes nothing and restarts nothing.
468
+ - It refuses, and says why, when the shared backend is a remote endpoint (browsing
469
+ text would leave the machine) or MinSync's in-process engine (Hister cannot reach
470
+ it; run `minsync serve --idle-timeout 0`, docs/minsync-setup.md).
471
+ - If `restartRequired` or `reindexRequired` is true in the report, the step was
472
+ not done (no Homebrew service, or it failed): run `brew services restart hister`
473
+ and `hister reindex` yourself. Check with `hister doctor`: the line
474
+ `server.index.embeddings` must be `ok`.
475
+ - A changed `query_prefix` alone needs a restart but no reindex. Never hand-edit the
476
+ `query_prefix` value across two lines: Hister's YAML parser folds the newline into
477
+ a space and the model's instruction is silently corrupted. `--sync-hister` writes
478
+ it correctly.
479
+
421
480
  ## Verify and build indexes
422
481
 
423
482
  Configuration alone is not a successful setup:
@@ -483,6 +542,8 @@ is set up.
483
542
  - `AUTORAG_MEMORY_PATH`
484
543
  - `AUTORAG_MODEL_PROVIDER`
485
544
  - `AUTORAG_MODEL_ID`
545
+ - `AUTORAG_PERMISSION_MODE` (`auto` default for the TUI, `full` for `autorag search` and a TUI with a chat bridge, `ask`, `full`; config key `permissionMode`; with `auto` Jev checks the agent's
546
+ file-changing calls and asks the user only when it will not approve, `ask` asks about every one, `full` never asks)
486
547
 
487
548
  ## Completion condition
488
549
 
@@ -1,15 +0,0 @@
1
- import type { QmdEmbedResult, QmdEnsureResult, QmdOptions, QmdSearchMode, QmdSearchOptions, QmdSearchResult, QmdUpdateResult } from "./types.js";
2
- /**
3
- * Thin external `qmd` CLI wrapper for Obsidian vault indexing/search.
4
- * Every method spawns the `qmd` binary, never throws for expected failures,
5
- * and isolates config/cache under the AutoRAG workspace.
6
- */
7
- export declare class QmdClient {
8
- private readonly options;
9
- constructor(options?: QmdOptions);
10
- ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
11
- update(signal?: AbortSignal): Promise<QmdUpdateResult>;
12
- embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
13
- search(mode: QmdSearchMode, query: string, options?: QmdSearchOptions): Promise<QmdSearchResult>;
14
- private run;
15
- }
@@ -1,92 +0,0 @@
1
- import type { RetrievalOptions } from "../../../retrieval/types.js";
2
- /**
3
- * Search modes accepted by the external `qmd` CLI.
4
- *
5
- * - `search` — BM25 / FTS5 lexical search
6
- * - `vsearch` — dense vector semantic search
7
- */
8
- export type QmdSearchMode = "search" | "vsearch";
9
- /**
10
- * Configuration for {@link QmdClient}. The client spawns the `qmd` binary as a
11
- * child process and isolates config/cache under the AutoRAG workspace.
12
- */
13
- export interface QmdOptions {
14
- /** Explicit path to the `qmd` binary. Defaults to a bare `qmd` PATH lookup. */
15
- readonly binaryPath?: string;
16
- /** Spawn timeout in milliseconds. Default 60_000 (embed can be slow). */
17
- readonly timeoutMs?: number;
18
- /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
19
- readonly maxBufferBytes?: number;
20
- /** Absolute path to the Obsidian vault. Required for ensure/update. */
21
- readonly vaultPath?: string;
22
- /** AutoRAG workspace root used to isolate qmd config/cache. */
23
- readonly workspaceRoot?: string;
24
- /** Instance id used for source paths and collection naming. Default `default`. */
25
- readonly instanceId?: string;
26
- /** Explicit collection name; defaults to a sanitized instance id. */
27
- readonly collectionName?: string;
28
- /** Environment overrides merged on top of a restricted process env. */
29
- readonly env?: Readonly<Record<string, string | undefined>>;
30
- /** Explicit operator-owned qmd config directory, never rewritten. */
31
- readonly configPath?: string;
32
- }
33
- export declare const DEFAULT_QMD_BINARY = "qmd";
34
- export declare const DEFAULT_QMD_TIMEOUT_MS = 60000;
35
- export declare const DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
36
- export type QmdFailureReason = "binary-missing" | "not-configured" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
37
- export interface QmdFailure {
38
- readonly ok: false;
39
- readonly reason: QmdFailureReason;
40
- readonly stdout: string;
41
- readonly stderr: string;
42
- readonly code: number | null;
43
- }
44
- export interface QmdOk<T> {
45
- readonly ok: true;
46
- readonly data: T;
47
- readonly stdout: string;
48
- readonly stderr: string;
49
- readonly code: number;
50
- }
51
- export interface QmdUpdateInfo {
52
- readonly indexed: number;
53
- readonly updated: number;
54
- readonly unchanged: number;
55
- readonly removed: number;
56
- readonly needsEmbedding?: boolean;
57
- readonly metadata?: Readonly<Record<string, unknown>>;
58
- }
59
- export interface QmdEmbedInfo {
60
- readonly embedded: boolean;
61
- readonly metadata?: Readonly<Record<string, unknown>>;
62
- }
63
- export interface QmdEnsureInfo {
64
- readonly collectionName: string;
65
- readonly vaultPath: string;
66
- readonly configDir: string;
67
- }
68
- /** A scored search hit from `qmd search` / `qmd vsearch`. */
69
- export interface QmdSearchHit {
70
- readonly chunkId: string;
71
- readonly score: number;
72
- readonly content: string;
73
- readonly title?: string;
74
- readonly file?: string;
75
- readonly docid?: string;
76
- readonly metadata?: Readonly<Record<string, unknown>>;
77
- }
78
- export interface QmdSearchOk {
79
- readonly ok: true;
80
- readonly hits: readonly QmdSearchHit[];
81
- readonly stdout: string;
82
- readonly stderr: string;
83
- readonly code: number;
84
- readonly data: {
85
- readonly hits: readonly QmdSearchHit[];
86
- };
87
- }
88
- export type QmdEnsureResult = QmdOk<QmdEnsureInfo> | QmdFailure;
89
- export type QmdUpdateResult = QmdOk<QmdUpdateInfo> | QmdFailure;
90
- export type QmdEmbedResult = QmdOk<QmdEmbedInfo> | QmdFailure;
91
- export type QmdSearchResult = QmdSearchOk | QmdFailure;
92
- export type QmdSearchOptions = RetrievalOptions;