@autorag/librarian 2.10.6 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +14 -0
- package/README.md +52 -10
- package/dist/agent/agent.d.ts +70 -2
- package/dist/agent/answer-citations.d.ts +34 -2
- package/dist/agent/answer-guidelines.d.ts +24 -3
- package/dist/agent/ask-user.d.ts +8 -0
- package/dist/agent/automation-tool.d.ts +45 -0
- package/dist/agent/citation-links.d.ts +23 -0
- package/dist/agent/citation-numbers.d.ts +68 -0
- package/dist/agent/citation-relevance.d.ts +37 -0
- package/dist/agent/conversation-queries.d.ts +31 -0
- package/dist/agent/convert-tool.d.ts +28 -0
- package/dist/agent/evidence-persistence.d.ts +17 -11
- package/dist/agent/index.d.ts +1 -0
- package/dist/agent/intent-tracing.d.ts +33 -0
- package/dist/agent/mcp-client.d.ts +27 -0
- package/dist/agent/permission-gate.d.ts +78 -0
- package/dist/agent/permission-judgment.d.ts +72 -0
- package/dist/agent/permission-paths.d.ts +23 -0
- package/dist/agent/pi-session.d.ts +8 -0
- package/dist/agent/query-routing.d.ts +14 -17
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/agent/search-progress.d.ts +96 -0
- package/dist/agent/self-config.d.ts +2 -0
- package/dist/agent/user-requests.d.ts +52 -0
- package/dist/agent/welcome-header.d.ts +35 -0
- package/dist/automation/config.d.ts +48 -0
- package/dist/automation/daemon.d.ts +31 -0
- package/dist/automation/dispatch.d.ts +13 -0
- package/dist/automation/run-agent.d.ts +22 -0
- package/dist/automation/run.d.ts +32 -0
- package/dist/automation/service.d.ts +61 -0
- package/dist/cli/commands/automate.d.ts +2 -0
- package/dist/cli/commands/setup.d.ts +2 -0
- package/dist/cli/config.d.ts +21 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +26740 -21094
- package/dist/cli/open-target.d.ts +9 -0
- package/dist/cli/setup.d.ts +21 -0
- package/dist/convert/cli.d.ts +77 -0
- package/dist/convert/index.d.ts +4 -0
- package/dist/convert/installer.d.ts +84 -0
- package/dist/convert/matrix.d.ts +27 -0
- package/dist/convert/relate.d.ts +30 -0
- package/dist/core.js +32095 -27288
- package/dist/datasource/connector-timeouts.d.ts +6 -2
- package/dist/datasource/crawler-types.d.ts +1 -1
- package/dist/datasource/errors.d.ts +1 -1
- package/dist/datasource/index.d.ts +2 -1
- package/dist/datasource/skills/factory.d.ts +3 -0
- package/dist/datasource/skills/hister/client.d.ts +83 -0
- package/dist/datasource/skills/hister/embedding.d.ts +64 -0
- package/dist/datasource/skills/hister/index.d.ts +2 -0
- package/dist/datasource/skills/hister/setup.d.ts +87 -0
- package/dist/datasource/skills/hister/skill.d.ts +63 -0
- package/dist/datasource/skills/lazykatok/types.d.ts +2 -2
- package/dist/datasource/skills/mailcrawl/types.d.ts +1 -1
- package/dist/datasource/skills/obsidian/index.d.ts +4 -6
- package/dist/datasource/skills/obsidian/methods.d.ts +13 -17
- package/dist/datasource/skills/obsidian/paths.d.ts +24 -24
- package/dist/datasource/skills/obsidian/skill.d.ts +10 -15
- package/dist/datasource/skills/obsidian/staging.d.ts +50 -0
- package/dist/datasource/skills/obsidian/vault-index.d.ts +44 -0
- package/dist/dupey/cli.d.ts +18 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +32263 -27429
- package/dist/mcp/index.js +32149 -27342
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/workspace-lock.d.ts +7 -0
- package/dist/process/terminate-tree.d.ts +4 -5
- package/dist/profiles/agent-factory.d.ts +60 -0
- package/dist/profiles/automation-profiles.d.ts +25 -0
- package/dist/profiles/loop-guard.d.ts +23 -0
- package/dist/profiles/retrieval-tools.d.ts +24 -0
- package/dist/profiles/session-model.d.ts +69 -0
- package/dist/profiles.d.ts +13 -0
- package/dist/profiles.js +34739 -0
- package/dist/retrieval/rerank.d.ts +6 -0
- package/licenses/hister-AGPL-3.0-notice.txt +47 -0
- package/package.json +9 -3
- package/skills/autorag/SKILL.md +79 -5
- package/skills/autorag-doctor/SKILL.md +5 -4
- package/skills/autorag-lite-setup/SKILL.md +23 -5
- package/skills/autorag-setup/SKILL.md +68 -7
- package/dist/datasource/skills/obsidian/client.d.ts +0 -15
- package/dist/datasource/skills/obsidian/types.d.ts +0 -92
|
@@ -33,6 +33,12 @@ export declare const DEFAULT_DAZZI_RERANK_MODEL = "qwen/qwen3-reranker-8b";
|
|
|
33
33
|
export declare const DEFAULT_RERANK_TIMEOUT_MS = 60000;
|
|
34
34
|
/** Default number of merged chunks kept after reranking. */
|
|
35
35
|
export declare const DEFAULT_RERANK_TOP_N = 25;
|
|
36
|
+
/**
|
|
37
|
+
* Longest document, in UTF-16 units, sent to a reranker. The hosted Dazzi
|
|
38
|
+
* reranker rejects a document of 131,072 characters or more with HTTP 400 and
|
|
39
|
+
* fails the whole request; 131,000 stays under that with a small margin.
|
|
40
|
+
*/
|
|
41
|
+
export declare const RERANK_MAX_DOCUMENT_CHARS = 131000;
|
|
36
42
|
/** Rerank providers {@link createReranker} can build. */
|
|
37
43
|
export declare const SUPPORTED_RERANK_PROVIDERS: readonly string[];
|
|
38
44
|
/** Default wire model for a rerank provider. */
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
Hister — license notice for AutoRAG's `hister` datasource
|
|
2
|
+
==========================================================
|
|
3
|
+
|
|
4
|
+
Program: Hister ("Your own search engine")
|
|
5
|
+
Source: https://github.com/asciimoo/hister
|
|
6
|
+
Copyright: asciimoo and Hister contributors (see the repository history)
|
|
7
|
+
License: GNU Affero General Public License, version 3 or (at your option)
|
|
8
|
+
any later version (AGPL-3.0-or-later)
|
|
9
|
+
License text: https://www.gnu.org/licenses/agpl-3.0.txt
|
|
10
|
+
(also the LICENSE file in the Hister repository)
|
|
11
|
+
|
|
12
|
+
How AutoRAG relates to Hister
|
|
13
|
+
-----------------------------
|
|
14
|
+
* AutoRAG does NOT contain, bundle, link, vendor, or redistribute any Hister
|
|
15
|
+
source code or binary. Nothing from the Hister repository is copied into
|
|
16
|
+
this package.
|
|
17
|
+
* Users install Hister themselves, unmodified, from its own releases
|
|
18
|
+
(https://github.com/asciimoo/hister/releases, `brew install hister`, Docker,
|
|
19
|
+
or Nix), and run it as a separate process (`hister listen`).
|
|
20
|
+
* AutoRAG talks to that process only over HTTP, using Hister's public JSON API
|
|
21
|
+
(`GET /search`, `GET /api/config`), the same API Hister's own `hister search`
|
|
22
|
+
command uses. Hister owns its index, its browser extension, its credentials,
|
|
23
|
+
and its optional embedding calls.
|
|
24
|
+
* AutoRAG does not modify Hister, and it never hides that Hister is the backend:
|
|
25
|
+
the datasource manifest shown to the agent names Hister, its license, and its
|
|
26
|
+
source URL.
|
|
27
|
+
* The `hister` datasource only reads web-page documents (`type:web`). It never
|
|
28
|
+
enables or uses Hister's local-directory indexing.
|
|
29
|
+
|
|
30
|
+
Combining separate programs that communicate over a network is "mere
|
|
31
|
+
aggregation" in the sense of the GNU licenses FAQ
|
|
32
|
+
(https://www.gnu.org/licenses/gpl-faq.html#MereAggregation); AutoRAG therefore
|
|
33
|
+
stays under its own license (Apache-2.0).
|
|
34
|
+
|
|
35
|
+
If you run Hister as a network service for other people, AGPL-3.0 section 13
|
|
36
|
+
applies to YOU as the operator of that Hister instance: you must offer those
|
|
37
|
+
users the Corresponding Source of the Hister version you run (and of any
|
|
38
|
+
modifications you made). Running an unmodified upstream release satisfies this
|
|
39
|
+
by pointing users at https://github.com/asciimoo/hister.
|
|
40
|
+
|
|
41
|
+
Obtaining the Hister source for a release you run
|
|
42
|
+
-------------------------------------------------
|
|
43
|
+
git clone https://github.com/asciimoo/hister
|
|
44
|
+
git checkout <the version printed by `hister --version`>
|
|
45
|
+
|
|
46
|
+
This file is a notice, not the license. The authoritative license terms are in
|
|
47
|
+
the Hister repository and at the AGPL URL above.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@autorag/librarian",
|
|
3
|
-
"version": "2.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "2.12.0",
|
|
4
|
+
"description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -18,6 +18,10 @@
|
|
|
18
18
|
"./core": {
|
|
19
19
|
"types": "./dist/core.d.ts",
|
|
20
20
|
"default": "./dist/core.js"
|
|
21
|
+
},
|
|
22
|
+
"./profiles": {
|
|
23
|
+
"types": "./dist/profiles.d.ts",
|
|
24
|
+
"default": "./dist/profiles.js"
|
|
21
25
|
}
|
|
22
26
|
},
|
|
23
27
|
"bin": {
|
|
@@ -36,7 +40,7 @@
|
|
|
36
40
|
"access": "public"
|
|
37
41
|
},
|
|
38
42
|
"scripts": {
|
|
39
|
-
"build": "bun build src/index.ts src/core.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
|
|
43
|
+
"build": "bun build src/index.ts src/core.ts src/profiles.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
|
|
40
44
|
"prepublishOnly": "bun run typecheck && bun run build",
|
|
41
45
|
"prepare": "bun run build",
|
|
42
46
|
"test": "node scripts/run-test-files.mjs",
|
|
@@ -56,6 +60,7 @@
|
|
|
56
60
|
"@earendil-works/pi-tui": "1.2.0",
|
|
57
61
|
"@llblab/pi-telegram": "0.57.2",
|
|
58
62
|
"@modelcontextprotocol/server": "^2.3.1",
|
|
63
|
+
"@parcel/watcher": "2.6.0",
|
|
59
64
|
"chardet": "^2.2.0",
|
|
60
65
|
"discord.js": "14.27.0",
|
|
61
66
|
"fast-xml-parser": "^5.11.2",
|
|
@@ -65,6 +70,7 @@
|
|
|
65
70
|
"kordoc": "^4.21.30",
|
|
66
71
|
"mailparser": "^3.9.38",
|
|
67
72
|
"pdfjs-dist": "^6.4.299",
|
|
73
|
+
"pi-ask-user": "0.17.0",
|
|
68
74
|
"pi-discord-remote": "0.2.4",
|
|
69
75
|
"puppeteer-core": "^25.13.0",
|
|
70
76
|
"slides-grab": "^1.5.2",
|
package/skills/autorag/SKILL.md
CHANGED
|
@@ -15,10 +15,13 @@ search, calls MinSync, Jikji, datasource, and filesystem tools, reads
|
|
|
15
15
|
sources, judges evidence, and curates the final answer. There is no subagent or
|
|
16
16
|
separate model role.
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
Indexing and refresh read source documents and write only under the configured
|
|
19
19
|
workspace `.autorag/` directory and Jikji's central store (`<OS data dir>/jikji/`,
|
|
20
|
-
plus a `.jikji_agent_map.md` guide in each indexed root).
|
|
21
|
-
|
|
20
|
+
plus a `.jikji_agent_map.md` guide in each indexed root). The librarian may
|
|
21
|
+
organize, create, or modify the user's own files in local sessions when the
|
|
22
|
+
user asks; it never deletes permanently (OS trash only) and lists every change
|
|
23
|
+
as old path → new path. Do not move, rename, edit, or delete source files on your own
|
|
24
|
+
initiative while searching or diagnosing.
|
|
22
25
|
|
|
23
26
|
## Preflight
|
|
24
27
|
|
|
@@ -42,6 +45,34 @@ near/contains families require human review. Exact duplicate exclusion during
|
|
|
42
45
|
refresh is enabled by default and can be disabled with
|
|
43
46
|
`"excludeExactDuplicates": false` in `config.json`.
|
|
44
47
|
|
|
48
|
+
### Converting files
|
|
49
|
+
|
|
50
|
+
The `convert_files` and `list_file_conversions` Agent tools run the
|
|
51
|
+
[Filee](https://github.com/KnifeLemon/Filee) command line to convert local
|
|
52
|
+
files offline: documents, spreadsheets, slides, HWP/HWPX, images, e-books,
|
|
53
|
+
archives, audio, video, data, and subtitles. They work on any path the user
|
|
54
|
+
names, not only the configured roots. This is the one exception to the
|
|
55
|
+
"never write source files" rule above: a conversion writes a new file and
|
|
56
|
+
never touches its source. An existing output name gets a number added unless
|
|
57
|
+
the user asks to replace it, in which case pass `overwrite: true`. Never
|
|
58
|
+
delete the source unless the user asked.
|
|
59
|
+
|
|
60
|
+
Each converted file is checked with `dupey compare` against its source, using
|
|
61
|
+
dupey's own thresholds, so the report says whether a later scan will group
|
|
62
|
+
them: `exact` (same canonical text, for example DOCX to HWPX), `near` (same
|
|
63
|
+
document in another format, for example Markdown to TXT), `different` (text
|
|
64
|
+
differs, typically layout formats such as PDF), `unsupported` (dupey cannot
|
|
65
|
+
read the format, for example HTML or images), or `unavailable` (dupey not
|
|
66
|
+
installed). The command line is `filee.binaryPath` if set, else a `filee`/`filee-cli`
|
|
67
|
+
on `PATH`, else the pinned [filee-cli](https://github.com/NomaDamas/filee-cli)
|
|
68
|
+
release that `autorag setup` (or the first conversion) downloads into
|
|
69
|
+
`<AUTORAG_HOME>/tools/filee-cli/`; `"filee": { "autoInstall": false }` forbids that
|
|
70
|
+
download. The system prompt carries the conversion map (which format family
|
|
71
|
+
converts to which, and the two-step chain when no direct route exists), so plan
|
|
72
|
+
from it instead of probing. `"filee": { "enabled": false }` in `config.json` removes both
|
|
73
|
+
tools; `filee.timeoutMs` (default 30 minutes) and `filee.dataDir` (Filee's
|
|
74
|
+
own settings and engines, `FILEE_DATA_DIR`) tune them.
|
|
75
|
+
|
|
45
76
|
### Excluding local files from the index
|
|
46
77
|
|
|
47
78
|
`"excludePaths"` in `config.json` lists files or folders (absolute, or relative
|
|
@@ -80,7 +111,10 @@ autorag search "what were the key findings in the Q3 report" --top-k 5 --json --
|
|
|
80
111
|
only `answer` plus `results`. Every
|
|
81
112
|
bracketed `[n]` citation in `answer` resolves to a `results[].number` of the
|
|
82
113
|
same response; an unmatched citation is removed and reported as a
|
|
83
|
-
`citation-without-result` diagnostic.
|
|
114
|
+
`citation-without-result` diagnostic. Within one conversation (an interactive
|
|
115
|
+
session, including a resumed one), a source keeps the `number` it was first
|
|
116
|
+
shown with and a new source gets the next unused number, so `results[].number`
|
|
117
|
+
can skip numbers and need not start at 1 on a later turn.
|
|
84
118
|
|
|
85
119
|
To inspect the exact persisted evidence behind numbered results, use:
|
|
86
120
|
|
|
@@ -113,7 +147,13 @@ default, OpenRouter), Jev first routes the question:
|
|
|
113
147
|
datasources) is handled by the agent itself: it loads the setup skill, edits
|
|
114
148
|
the active config, verifies it, and reports what changed. The running agent
|
|
115
149
|
keeps its startup model; changes apply to the next `autorag` launch.
|
|
116
|
-
-
|
|
150
|
+
- A request to act on files or folders (move, bulk rename, rename by content,
|
|
151
|
+
convert, sort) is carried out in one tool turn with no retrieval and no
|
|
152
|
+
verification, and reports every change as old path → new path.
|
|
153
|
+
- Anything else that needs no retrieval (commit, push, open a PR, run a build)
|
|
154
|
+
is carried out in one tool turn the same way.
|
|
155
|
+
- Private-data questions use local search; the agent decides on web search
|
|
156
|
+
after the fast answer.
|
|
117
157
|
- A multi-part question is split into up to five parallel search queries.
|
|
118
158
|
- On local search, Jev also picks which registered datasources (Slack,
|
|
119
159
|
Discord, KakaoTalk, email, ...) to search before the fast answer, from each
|
|
@@ -142,8 +182,42 @@ autorag index rebuild --yes --json
|
|
|
142
182
|
autorag index reset --method parsed --yes --json
|
|
143
183
|
autorag memory inspect --json
|
|
144
184
|
autorag tui
|
|
185
|
+
autorag automate status --json
|
|
186
|
+
autorag automate install --json
|
|
187
|
+
autorag automate uninstall --json
|
|
188
|
+
autorag automate run invoice --event create:/path/to/file.pdf --json
|
|
189
|
+
autorag automate daemon
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
File-triggered automations live in `~/.autorag/automations.json` (`automations`
|
|
193
|
+
array of `{ id, paths, events, include, exclude, run | agent, appendFiles, debounceMs, timeoutMs }`).
|
|
194
|
+
Edit `automations.json` directly to add or change rules; matched `[{ type, path }]`
|
|
195
|
+
events are piped on stdin to `run`. Use `autorag automate install` to register
|
|
196
|
+
the background daemon with launchd, `systemd --user`, or Windows Task Scheduler.
|
|
197
|
+
|
|
198
|
+
A rule runs exactly one of `run` (a command) or `agent` (an AI agent started in
|
|
199
|
+
the daemon on each batch of changes):
|
|
200
|
+
|
|
201
|
+
```json
|
|
202
|
+
{ "id": "inbox-triage", "paths": ["~/Downloads"], "include": ["**/*.pdf"],
|
|
203
|
+
"agent": { "profile": "file-triage", "task": "Summarize each new PDF.",
|
|
204
|
+
"retrieval": { "searchAll": true, "datasources": ["kakao"] } } }
|
|
145
205
|
```
|
|
146
206
|
|
|
207
|
+
`agent.profile` names a profile defined in code: `file-triage` (reads, lists and
|
|
208
|
+
searches files; changes nothing) or `file-organizer` (also `bash`, `edit`,
|
|
209
|
+
`write`). A rule never carries a prompt or a tool list, so the rules file cannot
|
|
210
|
+
grant an agent more than its profile has. `retrieval` optionally adds
|
|
211
|
+
`search_all_documents` and one `search_datasource_<id>` per listed datasource.
|
|
212
|
+
Retrieval memory is never connected to an automation agent.
|
|
213
|
+
|
|
214
|
+
Every agent run gets a loop guard appended to its system prompt, naming the
|
|
215
|
+
rule's folders, file types and events, and forbidding any change that would
|
|
216
|
+
fire the rule again, editing the rules file, and running `autorag automate`. At
|
|
217
|
+
most one agent run is in flight across all rules, and an agent rule is
|
|
218
|
+
disabled after 6 runs within 60 seconds (command rules: 20). The run's final
|
|
219
|
+
reply is the `stdout` of its log entry.
|
|
220
|
+
|
|
147
221
|
Prefer a full refresh so parsed mirrors, MinSync, Jikji, configured
|
|
148
222
|
datasources, and (on Windows) the bundled Everything file-name index stay
|
|
149
223
|
aligned. `--method` accepts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: autorag-doctor
|
|
3
|
-
description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl,
|
|
3
|
+
description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
|
|
4
4
|
license: Apache-2.0
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -15,9 +15,10 @@ Always finish with the status table in [Report](#report).
|
|
|
15
15
|
|
|
16
16
|
## Safety
|
|
17
17
|
|
|
18
|
-
-
|
|
18
|
+
- Diagnosis and repair never delete, move, or edit source documents; a doctor
|
|
19
|
+
run changes only AutoRAG-owned state.
|
|
19
20
|
- Never delete a datasource's **native** store (`~/Library/Application Support/katok`, `~/.discrawl`,
|
|
20
|
-
`~/.mailcrawl`,
|
|
21
|
+
`~/.mailcrawl`, Telegram/WhatsApp/Notion archives). AutoRAG only
|
|
21
22
|
reads them; rebuilding them is the owning CLI's job.
|
|
22
23
|
- Only AutoRAG-owned state under `AUTORAG_HOME` and the workspace `.autorag`
|
|
23
24
|
directory may be reset.
|
|
@@ -69,7 +70,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
|
|
|
69
70
|
| WhatsApp | `wacrawl --json doctor` | `wacrawl import` |
|
|
70
71
|
| Telegram | `telecrawl --json doctor`, `telecrawl --json status` | `telecrawl import` |
|
|
71
72
|
| Notion | `notcrawl doctor`, `notcrawl status` | `notcrawl sync --source desktop` |
|
|
72
|
-
| Obsidian / notes | `
|
|
73
|
+
| Obsidian / notes | `autorag status` (AutoRAG-owned index) | `autorag refresh --method datasources`; if corrupt, delete `.autorag/datasources/obsidian/<instance>` first |
|
|
73
74
|
| Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
|
|
74
75
|
| Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
|
|
75
76
|
| Spotlight (macOS) | `mdutil -s /` | indexed by the OS; no AutoRAG sync |
|
|
@@ -17,9 +17,9 @@ Use `autorag-setup` instead when the model-backed librarian must be configured.
|
|
|
17
17
|
`memoryPath`, `minSync`, `jikji`, `datasources`.
|
|
18
18
|
- Never print, copy, migrate, compare, or persist credential values. Store only
|
|
19
19
|
environment-variable names such as `tokenEnv` or `apiKeyEnv`.
|
|
20
|
-
-
|
|
21
|
-
indexes only under the configured workspace `.autorag/` directory
|
|
22
|
-
Jikji's central store (`<OS data dir>/jikji/`).
|
|
20
|
+
- Setup and Lite commands never move, rename, edit, or delete source documents.
|
|
21
|
+
They write indexes only under the configured workspace `.autorag/` directory
|
|
22
|
+
and Jikji's central store (`<OS data dir>/jikji/`).
|
|
23
23
|
- Do not index system trees, app bundles, caches, credential stores,
|
|
24
24
|
`node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
|
|
25
25
|
`.jikji`.
|
|
@@ -113,7 +113,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
|
|
|
113
113
|
|
|
114
114
|
1. Probe every datasource for setup feasibility before asking the user
|
|
115
115
|
anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
|
|
116
|
-
`wacrawl`, `telecrawl`, `notcrawl`, `
|
|
116
|
+
`wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`) and its
|
|
117
117
|
local store or archive is present. CLI-backed datasources own their own
|
|
118
118
|
archive, index, and authentication, so environment credentials (such as bot
|
|
119
119
|
tokens) are never required or checked for them. Non-CLI connectors
|
|
@@ -137,7 +137,7 @@ Datasource skills belong in trusted config.
|
|
|
137
137
|
|
|
138
138
|
Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
|
|
139
139
|
`slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
|
|
140
|
-
`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`) or connection
|
|
140
|
+
`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, `hister`) or connection
|
|
141
141
|
aliases with `"type": "<template>"`. Unknown names are skipped with an
|
|
142
142
|
`unknown-datasource-skill` warning; they do not fail config resolution.
|
|
143
143
|
`scope` narrows a query to a sub-path as ordinary filtering. Tags are
|
|
@@ -146,6 +146,24 @@ select configured connections before retrieval; discover their IDs with
|
|
|
146
146
|
`autorag.datasources.list`. Store only env-var names such as `tokenEnv` or
|
|
147
147
|
`apiKeyEnv`, never credential values.
|
|
148
148
|
|
|
149
|
+
### Browser history (Hister) and the shared embedding model
|
|
150
|
+
|
|
151
|
+
`hister` needs a separately installed, always-running Hister server (AGPL-3.0, never
|
|
152
|
+
bundled by AutoRAG). Run `autorag setup --format json` and read the `hister` object
|
|
153
|
+
(`installed`, `running`, `embedding`); each problem carries the exact command to run:
|
|
154
|
+
`brew install hister`, `brew services start hister`, then install the Hister browser
|
|
155
|
+
extension in the user's browser.
|
|
156
|
+
|
|
157
|
+
Hister embeds with its own `semantic_search` config, so it must use the same model as
|
|
158
|
+
AutoRAG (the `minsync serve` daemon, `Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions) or
|
|
159
|
+
its vectors stop lining up without any error. `autorag setup --sync-hister` copies the
|
|
160
|
+
shared settings into Hister's `config.yml` (backup kept), restarts the service, waits
|
|
161
|
+
for it, and runs `hister reindex` when needed; it is idempotent, and it refuses a
|
|
162
|
+
remote or in-process backend. Re-run it after restarting the `minsync serve` daemon
|
|
163
|
+
(its port changes). Verify with `hister doctor`: `server.index.embeddings` must be `ok`.
|
|
164
|
+
Install a skip list in Hister's `rules.json` before importing past history, because the
|
|
165
|
+
importer re-downloads URLs without the browser's cookies (docs/datasource-skills.md).
|
|
166
|
+
|
|
149
167
|
## Build and refresh indexes
|
|
150
168
|
|
|
151
169
|
After connecting, call MCP `autorag.refresh` with `{}` for an incremental run.
|
|
@@ -21,7 +21,8 @@ librarian, not the Lite MCP server.
|
|
|
21
21
|
- Never print, copy, migrate, compare, or persist credential values. Store only
|
|
22
22
|
environment-variable names such as `apiKeyEnv`.
|
|
23
23
|
- Do not scan the whole filesystem or home directory without explicit approval.
|
|
24
|
-
-
|
|
24
|
+
- Setup, indexing, and refresh never move, rename, edit, or delete source
|
|
25
|
+
documents. The librarian itself may organize files when the user asks it to.
|
|
25
26
|
- Do not index system trees, app bundles, caches, credential stores,
|
|
26
27
|
`node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
|
|
27
28
|
`.jikji`.
|
|
@@ -200,7 +201,7 @@ resolves in order:
|
|
|
200
201
|
|
|
201
202
|
1. A configured remote endpoint (`minSync.embedder`), used for everything; no
|
|
202
203
|
daemon is started and no local model is loaded.
|
|
203
|
-
2. A running `minsync serve` daemon (loopback only; MinSync 0.4.
|
|
204
|
+
2. A running `minsync serve` daemon (loopback only; MinSync 0.4.12+).
|
|
204
205
|
AutoRAG never starts it: run `minsync serve --idle-timeout 0` yourself, as
|
|
205
206
|
a launchd/systemd service (see `docs/minsync-setup.md`), or press
|
|
206
207
|
Ctrl+Shift+E in `autorag tui`. `autorag setup` only reports whether it is
|
|
@@ -238,9 +239,11 @@ fast and multi-part questions thorough.
|
|
|
238
239
|
- **Jev** (`jev`, default `{ "backend": "dazzi" }` when signed in to Dazzi
|
|
239
240
|
Models, else `{ "backend": "openrouter" }`; model `jev-1.13` on Dazzi,
|
|
240
241
|
`typesafe/jev-1.13` on OpenRouter) runs before the fast answer. It routes each question to
|
|
241
|
-
local search,
|
|
242
|
-
talk skips retrieval entirely),
|
|
243
|
-
|
|
242
|
+
local search, a direct answer (general knowledge or small
|
|
243
|
+
talk skips retrieval entirely), `organize` (moving, renaming, converting, or
|
|
244
|
+
sorting files and folders), `extra` (any other action that needs no
|
|
245
|
+
retrieval, such as committing code), or the `config` branch, and decides
|
|
246
|
+
whether to decompose it. On
|
|
244
247
|
local search it also decides, per registered datasource, whether to search
|
|
245
248
|
it before the fast answer, using each datasource's `description` and where
|
|
246
249
|
similar past questions were answered (retrieval memory). When setting up a
|
|
@@ -361,7 +364,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
|
|
|
361
364
|
|
|
362
365
|
1. Probe every datasource for setup feasibility before asking the user
|
|
363
366
|
anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
|
|
364
|
-
`wacrawl`, `telecrawl`, `notcrawl`, `
|
|
367
|
+
`wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`, `lark-cli`) and its
|
|
365
368
|
local store or archive is present. CLI-backed datasources own their own
|
|
366
369
|
archive, index, and authentication, so environment credentials (such as bot
|
|
367
370
|
tokens) are never required or checked for them. Non-CLI connectors
|
|
@@ -384,7 +387,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
|
|
|
384
387
|
Datasource skills belong in trusted config. Builtin
|
|
385
388
|
template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
|
|
386
389
|
`clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
|
|
387
|
-
`mailcrawl`, `obsidian`, `rss`, `spotlight`, and `
|
|
390
|
+
`mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, and `hister`. Config keys may be connection
|
|
388
391
|
aliases with `"type": "<template>"`. Unknown names are skipped with an
|
|
389
392
|
`unknown-datasource-skill` warning; they do not fail config resolution.
|
|
390
393
|
`scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.
|
|
@@ -418,6 +421,62 @@ the next `index`. kakao needs `lazykatok` 0.3.3 or newer for the same reason.
|
|
|
418
421
|
Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
|
|
419
422
|
datasource is removed.
|
|
420
423
|
|
|
424
|
+
### Browser history (Hister) and the shared embedding model
|
|
425
|
+
|
|
426
|
+
`hister` is the one datasource whose backing program AutoRAG does not install,
|
|
427
|
+
start, or bundle (it is AGPL-3.0-or-later and stays a separate program). Setting
|
|
428
|
+
it up is an agent task with these steps. Run each command; do not just describe it.
|
|
429
|
+
|
|
430
|
+
1. **Probe first.** Add the datasource entry, then run `autorag setup --format json`
|
|
431
|
+
and read the `hister` object: `installed`, `running`, and `embedding`. Every
|
|
432
|
+
problem has a `remediation` string with the exact next command.
|
|
433
|
+
2. **Install Hister** when `installed` is false: `brew install hister` (other
|
|
434
|
+
platforms: the release binaries at https://github.com/asciimoo/hister/releases).
|
|
435
|
+
3. **Run it as a login item** when `running` is false: `brew services start hister`.
|
|
436
|
+
Hister must stay running: its browser extension only captures while it is up,
|
|
437
|
+
and a page visited while it is down is not re-sent. AutoRAG never starts it.
|
|
438
|
+
4. **Install the browser extension** (Chrome Web Store "Hister", or Firefox
|
|
439
|
+
Add-ons). It must be done in the user's browser; tell the user if you cannot.
|
|
440
|
+
5. **Share the embedding model** (below).
|
|
441
|
+
6. **Before importing past history, install a skip list** in Hister's `rules.json`
|
|
442
|
+
(login, mail, documents, social and console domains, private hosts, token-bearing
|
|
443
|
+
URLs). `hister import browser history` re-downloads each URL *without* the
|
|
444
|
+
browser's cookies. See docs/datasource-skills.md, "Browser history via Hister".
|
|
445
|
+
7. Add `"browser": { "type": "hister", "connector": { "semantic": true } }` under
|
|
446
|
+
`datasources`, then `autorag refresh` (only probes Hister) and
|
|
447
|
+
`autorag lite retrieve "<a question>" --scope '/browser/**' --json`.
|
|
448
|
+
|
|
449
|
+
**Shared embedding model.** Hister keeps its own vector store and embeds with
|
|
450
|
+
whatever its `config.yml` `semantic_search` block says. Pointing it at a different
|
|
451
|
+
model does not fail: vectors simply stop lining up and results get worse. So Hister
|
|
452
|
+
must use the same model as AutoRAG, which is the `minsync serve` daemon (model
|
|
453
|
+
`Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, with Qwen3's query instruction).
|
|
454
|
+
|
|
455
|
+
```bash
|
|
456
|
+
autorag setup --format json # reports hister.embedding.differs: the keys that disagree
|
|
457
|
+
autorag setup --sync-hister # writes them, restarts Hister, re-embeds pages if needed
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
- `--sync-hister` rewrites only `enable`, `embedding_endpoint`, `embedding_model`,
|
|
461
|
+
`dimensions`, `query_prefix`, `document_prefix` in Hister's config (a
|
|
462
|
+
`config.yml.bak.<time>` is kept next to it), then runs `brew services restart hister`,
|
|
463
|
+
waits for it to answer, and runs `hister reindex` when a change makes stored
|
|
464
|
+
vectors stale. Without the flag setup never writes Hister's config.
|
|
465
|
+
- The daemon port is chosen when `minsync serve` starts. After restarting the daemon,
|
|
466
|
+
run `autorag setup --sync-hister` again. It is idempotent: when nothing differs
|
|
467
|
+
it changes nothing and restarts nothing.
|
|
468
|
+
- It refuses, and says why, when the shared backend is a remote endpoint (browsing
|
|
469
|
+
text would leave the machine) or MinSync's in-process engine (Hister cannot reach
|
|
470
|
+
it; run `minsync serve --idle-timeout 0`, docs/minsync-setup.md).
|
|
471
|
+
- If `restartRequired` or `reindexRequired` is true in the report, the step was
|
|
472
|
+
not done (no Homebrew service, or it failed): run `brew services restart hister`
|
|
473
|
+
and `hister reindex` yourself. Check with `hister doctor`: the line
|
|
474
|
+
`server.index.embeddings` must be `ok`.
|
|
475
|
+
- A changed `query_prefix` alone needs a restart but no reindex. Never hand-edit the
|
|
476
|
+
`query_prefix` value across two lines: Hister's YAML parser folds the newline into
|
|
477
|
+
a space and the model's instruction is silently corrupted. `--sync-hister` writes
|
|
478
|
+
it correctly.
|
|
479
|
+
|
|
421
480
|
## Verify and build indexes
|
|
422
481
|
|
|
423
482
|
Configuration alone is not a successful setup:
|
|
@@ -483,6 +542,8 @@ is set up.
|
|
|
483
542
|
- `AUTORAG_MEMORY_PATH`
|
|
484
543
|
- `AUTORAG_MODEL_PROVIDER`
|
|
485
544
|
- `AUTORAG_MODEL_ID`
|
|
545
|
+
- `AUTORAG_PERMISSION_MODE` (`auto` default for the TUI, `full` for `autorag search` and a TUI with a chat bridge, `ask`, `full`; config key `permissionMode`; with `auto` Jev checks the agent's
|
|
546
|
+
file-changing calls and asks the user only when it will not approve, `ask` asks about every one, `full` never asks)
|
|
486
547
|
|
|
487
548
|
## Completion condition
|
|
488
549
|
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { QmdEmbedResult, QmdEnsureResult, QmdOptions, QmdSearchMode, QmdSearchOptions, QmdSearchResult, QmdUpdateResult } from "./types.js";
|
|
2
|
-
/**
|
|
3
|
-
* Thin external `qmd` CLI wrapper for Obsidian vault indexing/search.
|
|
4
|
-
* Every method spawns the `qmd` binary, never throws for expected failures,
|
|
5
|
-
* and isolates config/cache under the AutoRAG workspace.
|
|
6
|
-
*/
|
|
7
|
-
export declare class QmdClient {
|
|
8
|
-
private readonly options;
|
|
9
|
-
constructor(options?: QmdOptions);
|
|
10
|
-
ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
|
|
11
|
-
update(signal?: AbortSignal): Promise<QmdUpdateResult>;
|
|
12
|
-
embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
|
|
13
|
-
search(mode: QmdSearchMode, query: string, options?: QmdSearchOptions): Promise<QmdSearchResult>;
|
|
14
|
-
private run;
|
|
15
|
-
}
|
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
import type { RetrievalOptions } from "../../../retrieval/types.js";
|
|
2
|
-
/**
|
|
3
|
-
* Search modes accepted by the external `qmd` CLI.
|
|
4
|
-
*
|
|
5
|
-
* - `search` — BM25 / FTS5 lexical search
|
|
6
|
-
* - `vsearch` — dense vector semantic search
|
|
7
|
-
*/
|
|
8
|
-
export type QmdSearchMode = "search" | "vsearch";
|
|
9
|
-
/**
|
|
10
|
-
* Configuration for {@link QmdClient}. The client spawns the `qmd` binary as a
|
|
11
|
-
* child process and isolates config/cache under the AutoRAG workspace.
|
|
12
|
-
*/
|
|
13
|
-
export interface QmdOptions {
|
|
14
|
-
/** Explicit path to the `qmd` binary. Defaults to a bare `qmd` PATH lookup. */
|
|
15
|
-
readonly binaryPath?: string;
|
|
16
|
-
/** Spawn timeout in milliseconds. Default 60_000 (embed can be slow). */
|
|
17
|
-
readonly timeoutMs?: number;
|
|
18
|
-
/** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
|
|
19
|
-
readonly maxBufferBytes?: number;
|
|
20
|
-
/** Absolute path to the Obsidian vault. Required for ensure/update. */
|
|
21
|
-
readonly vaultPath?: string;
|
|
22
|
-
/** AutoRAG workspace root used to isolate qmd config/cache. */
|
|
23
|
-
readonly workspaceRoot?: string;
|
|
24
|
-
/** Instance id used for source paths and collection naming. Default `default`. */
|
|
25
|
-
readonly instanceId?: string;
|
|
26
|
-
/** Explicit collection name; defaults to a sanitized instance id. */
|
|
27
|
-
readonly collectionName?: string;
|
|
28
|
-
/** Environment overrides merged on top of a restricted process env. */
|
|
29
|
-
readonly env?: Readonly<Record<string, string | undefined>>;
|
|
30
|
-
/** Explicit operator-owned qmd config directory, never rewritten. */
|
|
31
|
-
readonly configPath?: string;
|
|
32
|
-
}
|
|
33
|
-
export declare const DEFAULT_QMD_BINARY = "qmd";
|
|
34
|
-
export declare const DEFAULT_QMD_TIMEOUT_MS = 60000;
|
|
35
|
-
export declare const DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
|
|
36
|
-
export type QmdFailureReason = "binary-missing" | "not-configured" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
|
|
37
|
-
export interface QmdFailure {
|
|
38
|
-
readonly ok: false;
|
|
39
|
-
readonly reason: QmdFailureReason;
|
|
40
|
-
readonly stdout: string;
|
|
41
|
-
readonly stderr: string;
|
|
42
|
-
readonly code: number | null;
|
|
43
|
-
}
|
|
44
|
-
export interface QmdOk<T> {
|
|
45
|
-
readonly ok: true;
|
|
46
|
-
readonly data: T;
|
|
47
|
-
readonly stdout: string;
|
|
48
|
-
readonly stderr: string;
|
|
49
|
-
readonly code: number;
|
|
50
|
-
}
|
|
51
|
-
export interface QmdUpdateInfo {
|
|
52
|
-
readonly indexed: number;
|
|
53
|
-
readonly updated: number;
|
|
54
|
-
readonly unchanged: number;
|
|
55
|
-
readonly removed: number;
|
|
56
|
-
readonly needsEmbedding?: boolean;
|
|
57
|
-
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
58
|
-
}
|
|
59
|
-
export interface QmdEmbedInfo {
|
|
60
|
-
readonly embedded: boolean;
|
|
61
|
-
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
62
|
-
}
|
|
63
|
-
export interface QmdEnsureInfo {
|
|
64
|
-
readonly collectionName: string;
|
|
65
|
-
readonly vaultPath: string;
|
|
66
|
-
readonly configDir: string;
|
|
67
|
-
}
|
|
68
|
-
/** A scored search hit from `qmd search` / `qmd vsearch`. */
|
|
69
|
-
export interface QmdSearchHit {
|
|
70
|
-
readonly chunkId: string;
|
|
71
|
-
readonly score: number;
|
|
72
|
-
readonly content: string;
|
|
73
|
-
readonly title?: string;
|
|
74
|
-
readonly file?: string;
|
|
75
|
-
readonly docid?: string;
|
|
76
|
-
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
77
|
-
}
|
|
78
|
-
export interface QmdSearchOk {
|
|
79
|
-
readonly ok: true;
|
|
80
|
-
readonly hits: readonly QmdSearchHit[];
|
|
81
|
-
readonly stdout: string;
|
|
82
|
-
readonly stderr: string;
|
|
83
|
-
readonly code: number;
|
|
84
|
-
readonly data: {
|
|
85
|
-
readonly hits: readonly QmdSearchHit[];
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
export type QmdEnsureResult = QmdOk<QmdEnsureInfo> | QmdFailure;
|
|
89
|
-
export type QmdUpdateResult = QmdOk<QmdUpdateInfo> | QmdFailure;
|
|
90
|
-
export type QmdEmbedResult = QmdOk<QmdEmbedInfo> | QmdFailure;
|
|
91
|
-
export type QmdSearchResult = QmdSearchOk | QmdFailure;
|
|
92
|
-
export type QmdSearchOptions = RetrievalOptions;
|