@autorag/librarian 2.10.5 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/NOTICE +14 -9
  2. package/README.md +65 -25
  3. package/dist/agent/agent.d.ts +51 -49
  4. package/dist/agent/answer-guidelines.d.ts +22 -1
  5. package/dist/agent/context-token-guard.d.ts +11 -0
  6. package/dist/agent/discord-remote.d.ts +30 -0
  7. package/dist/agent/embedding-controls.d.ts +22 -0
  8. package/dist/agent/index.d.ts +1 -2
  9. package/dist/agent/mcp-client.d.ts +27 -0
  10. package/dist/agent/pi-session.d.ts +8 -1
  11. package/dist/agent/query-decomposition.d.ts +1 -1
  12. package/dist/agent/search-documents.d.ts +2 -2
  13. package/dist/agent/search-progress.d.ts +96 -0
  14. package/dist/agent/system-prompt.d.ts +1 -1
  15. package/dist/agent/welcome-header.d.ts +35 -0
  16. package/dist/cli/commands/models.d.ts +0 -14
  17. package/dist/cli/commands/search.d.ts +3 -4
  18. package/dist/cli/commands/setup.d.ts +2 -0
  19. package/dist/cli/config.d.ts +7 -19
  20. package/dist/cli/index.d.ts +1 -1
  21. package/dist/cli/index.js +16364 -18337
  22. package/dist/cli/setup.d.ts +22 -36
  23. package/dist/cloud/config.d.ts +3 -3
  24. package/dist/core.js +7372 -9206
  25. package/dist/datasource/embedding-env.d.ts +29 -0
  26. package/dist/datasource/errors.d.ts +1 -1
  27. package/dist/datasource/index.d.ts +2 -1
  28. package/dist/datasource/sidecar/config.d.ts +3 -0
  29. package/dist/datasource/sidecar/index.d.ts +1 -1
  30. package/dist/datasource/skills/discrawl/client.d.ts +1 -0
  31. package/dist/datasource/skills/discrawl/embedding.d.ts +17 -0
  32. package/dist/datasource/skills/discrawl/types.d.ts +11 -3
  33. package/dist/datasource/skills/factory.d.ts +17 -1
  34. package/dist/datasource/skills/github-gist/semantic.d.ts +5 -5
  35. package/dist/datasource/skills/github-gist/skill.d.ts +2 -3
  36. package/dist/datasource/skills/hister/client.d.ts +83 -0
  37. package/dist/datasource/skills/hister/embedding.d.ts +64 -0
  38. package/dist/datasource/skills/hister/index.d.ts +2 -0
  39. package/dist/datasource/skills/hister/setup.d.ts +87 -0
  40. package/dist/datasource/skills/hister/skill.d.ts +63 -0
  41. package/dist/datasource/skills/lazykatok/types.d.ts +9 -1
  42. package/dist/datasource/skills/mailcrawl/types.d.ts +9 -1
  43. package/dist/datasource/skills/obsidian/index.d.ts +4 -6
  44. package/dist/datasource/skills/obsidian/methods.d.ts +13 -17
  45. package/dist/datasource/skills/obsidian/paths.d.ts +24 -24
  46. package/dist/datasource/skills/obsidian/skill.d.ts +10 -15
  47. package/dist/datasource/skills/obsidian/staging.d.ts +50 -0
  48. package/dist/datasource/skills/obsidian/vault-index.d.ts +44 -0
  49. package/dist/embedding/daemon.d.ts +57 -0
  50. package/dist/embedding/embedder.d.ts +59 -0
  51. package/dist/embedding/http-embedder.d.ts +28 -0
  52. package/dist/embedding/in-process.d.ts +19 -0
  53. package/dist/embedding/index.d.ts +28 -0
  54. package/dist/embedding/managed.d.ts +44 -0
  55. package/dist/embedding/prefixes.d.ts +18 -0
  56. package/dist/embedding/settings.d.ts +9 -0
  57. package/dist/index.js +7465 -9298
  58. package/dist/mcp/index.js +7460 -9294
  59. package/dist/memory/context.d.ts +1 -1
  60. package/dist/memory/similar-queries.d.ts +1 -1
  61. package/dist/minsync/binary.d.ts +21 -0
  62. package/dist/minsync/client.d.ts +11 -10
  63. package/dist/minsync/index.d.ts +1 -1
  64. package/dist/minsync/installer.d.ts +1 -1
  65. package/dist/minsync/method.d.ts +3 -11
  66. package/dist/minsync/process.d.ts +1 -0
  67. package/dist/minsync/types.d.ts +0 -2
  68. package/dist/minsync/workspace-lock.d.ts +7 -0
  69. package/dist/process/terminate-tree.d.ts +4 -5
  70. package/dist/retrieval/rerank.d.ts +10 -3
  71. package/licenses/hister-AGPL-3.0-notice.txt +47 -0
  72. package/package.json +4 -4
  73. package/skills/autorag/SKILL.md +9 -11
  74. package/skills/autorag-doctor/SKILL.md +50 -29
  75. package/skills/autorag-lite-setup/SKILL.md +31 -13
  76. package/skills/autorag-setup/SKILL.md +99 -26
  77. package/dist/agent/peer-target-tool.d.ts +0 -19
  78. package/dist/agent/query-peer-tool.d.ts +0 -51
  79. package/dist/cli/commands/gateway.d.ts +0 -7
  80. package/dist/cli/commands/p2p-policy.d.ts +0 -2
  81. package/dist/cli/commands/p2p.d.ts +0 -7
  82. package/dist/cli/commands/serve.d.ts +0 -19
  83. package/dist/datasource/skills/obsidian/client.d.ts +0 -15
  84. package/dist/datasource/skills/obsidian/types.d.ts +0 -92
  85. package/dist/embedding-runtime/cache.d.ts +0 -30
  86. package/dist/embedding-runtime/gateway-embedder.d.ts +0 -41
  87. package/dist/embedding-runtime/gateway.d.ts +0 -27
  88. package/dist/embedding-runtime/index.d.ts +0 -120
  89. package/dist/embedding-runtime/manifest.d.ts +0 -20
  90. package/dist/embedding-runtime/supervisor.d.ts +0 -74
  91. package/dist/embedding-runtime/types.d.ts +0 -70
  92. package/dist/p2p/approval-store.d.ts +0 -32
  93. package/dist/p2p/egress-gate.d.ts +0 -22
  94. package/dist/p2p/file-sharing.d.ts +0 -33
  95. package/dist/p2p/injection-classifier.d.ts +0 -33
  96. package/dist/p2p/injection-gate.d.ts +0 -37
  97. package/dist/p2p/pii-gate.d.ts +0 -47
  98. package/dist/p2p/policy-filter.d.ts +0 -15
  99. package/dist/p2p/policy.d.ts +0 -77
  100. package/dist/p2p/simplex-query-store.d.ts +0 -44
  101. package/dist/p2p/simplex-server.d.ts +0 -115
  102. package/dist/p2p/simplex-transport.d.ts +0 -62
  103. package/dist/p2p/wire.d.ts +0 -39
  104. package/licenses/embedding-assets.json +0 -85
  105. package/licenses/llama.cpp-MIT.txt +0 -21
  106. package/licenses/qwen3-embedding-notice.txt +0 -13
@@ -21,7 +21,8 @@ librarian, not the Lite MCP server.
21
21
  - Never print, copy, migrate, compare, or persist credential values. Store only
22
22
  environment-variable names such as `apiKeyEnv`.
23
23
  - Do not scan the whole filesystem or home directory without explicit approval.
24
- - Never move, rename, edit, or delete source documents.
24
+ - Setup, indexing, and refresh never move, rename, edit, or delete source
25
+ documents. The librarian itself may organize files when the user asks it to.
25
26
  - Do not index system trees, app bundles, caches, credential stores,
26
27
  `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
27
28
  `.jikji`.
@@ -189,20 +190,28 @@ is impossible, refresh continues without this optimization and the user is
189
190
  told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
190
191
  index every copy.
191
192
 
192
- MinSync's default embedder is in-process native Qwen3 embeddings
193
- (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.10). The
194
- default needs no embedder flags, no API key, and no external daemon (such as
195
- Ollama) — all embeddings run in-process locally and privately. For workspaces
196
- using the loopback llama-server gateway, prefetch the verified model with
197
- `autorag models prefetch --profile qwen3-embedding-0.6b` (or verify with
198
- `autorag models verify --profile qwen3-embedding-0.6b`).
199
-
200
- The legacy Ollama/TEI adapter path (EmbeddingGemma, 768 dimensions) is
201
- supported only for existing legacy workspaces and manual QA; do not use it as a
202
- fresh install default.
203
-
204
- Override the embedder only when intentionally using a different, for example
205
- remote, provider:
193
+ AutoRAG owns no embedding model. MinSync provisions the embedder
194
+ (`Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions) in its own cache
195
+ (`~/.cache/minsync/models`); there is no local model download and no bundled
196
+ runtime. One shared embedder serves everything that embeds inside
197
+ AutoRAG: MinSync indexing/query, memory similar-question search, the
198
+ chat-crawler sidecars, the GitHub Gist vector index, and the endpoint external
199
+ CLIs are pointed at. The default needs no embedder flags and no API key. It
200
+ resolves in order:
201
+
202
+ 1. A configured remote endpoint (`minSync.embedder`), used for everything; no
203
+ daemon is started and no local model is loaded.
204
+ 2. A running `minsync serve` daemon (loopback only; MinSync 0.4.11+).
205
+ AutoRAG never starts it: run `minsync serve --idle-timeout 0` yourself, as
206
+ a launchd/systemd service (see `docs/minsync-setup.md`), or press
207
+ Ctrl+Shift+E in `autorag tui`. `autorag setup` only reports whether it is
208
+ alive.
209
+ 3. MinSync's in-process engine, through the optional Node-API addon
210
+ `@nomadamas/minsync-native` (same engine and weights, so bit-for-bit
211
+ identical vectors; not bundled). Without a daemon and without the addon,
212
+ embedding fails with an error naming the missing package.
213
+
214
+ Override the embedder only when intentionally using a remote provider:
206
215
 
207
216
  ```bash
208
217
  autorag init \
@@ -213,9 +222,14 @@ autorag init \
213
222
  --embedder-batch-size 64
214
223
  ```
215
224
 
216
- Only store the environment-variable name, never its value. Dimension and batch
217
- size must be positive integers, and the dimension must match the embedder
218
- (default Qwen3 is 1024; legacy EmbeddingGemma is 768; `voyageai/voyage-4-lite` via OpenRouter is 1024).
225
+ A remote `minSync.embedder` names an `id` (`openai:<model>`, `tei:<model>`, or
226
+ any id together with a `baseUrl`) plus optional `apiKeyEnv` (the **name** of the
227
+ environment variable holding the bearer token), `dimension`, `batchSize`, and
228
+ `timeoutMs`. OpenAI-protocol endpoints POST `{baseUrl}/v1/embeddings`; a `tei:`
229
+ endpoint POSTs `{baseUrl}/embed`; `openai:` with no `baseUrl` means
230
+ `https://api.openai.com`. Only store the environment-variable name, never its
231
+ value. `dimension` and `batchSize` must be positive integers, and `dimension`
232
+ must match the endpoint (the local default is 1024).
219
233
 
220
234
  ### Jev routing and question decomposition (on by default)
221
235
 
@@ -336,7 +350,7 @@ keeps its mirrors and reports `search-root-unavailable`; see
336
350
 
337
351
  ## Probe and configure datasource skills (setup wizard)
338
352
 
339
- Use `autorag setup` to probe the local runtime, model profile, and known
353
+ Use `autorag setup` to probe the embedding backend, MinSync, and known
340
354
  datasources automatically:
341
355
 
342
356
  ```bash
@@ -348,7 +362,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
348
362
 
349
363
  1. Probe every datasource for setup feasibility before asking the user
350
364
  anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
351
- `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`, `lark-cli`) and its
365
+ `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`, `lark-cli`) and its
352
366
  local store or archive is present. CLI-backed datasources own their own
353
367
  archive, index, and authentication, so environment credentials (such as bot
354
368
  tokens) are never required or checked for them. Non-CLI connectors
@@ -371,7 +385,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
371
385
  Datasource skills belong in trusted config. Builtin
372
386
  template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
373
387
  `clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
374
- `mailcrawl`, `obsidian`, `rss`, `spotlight`, and `lark`. Config keys may be connection
388
+ `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, and `hister`. Config keys may be connection
375
389
  aliases with `"type": "<template>"`. Unknown names are skipped with an
376
390
  `unknown-datasource-skill` warning; they do not fail config resolution.
377
391
  `scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.
@@ -393,16 +407,74 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
393
407
  Tokens are environment-variable names, not raw secrets. CLI-backed connectors
394
408
  keep authentication in their external tool configuration.
395
409
 
396
- Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.0` or newer)
410
+ Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.2` or newer)
397
411
  and configured through its own Himalaya account. AutoRAG runs its local `sync`
398
412
  and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
399
413
  hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
400
- fails with `text array must be non-empty`. 0.2.0 defaults to the in-process
401
- native `Qwen/Qwen3-Embedding-0.6B` embedder and keeps vectors in LanceDB, so a
402
- cold cache makes the first `index` download ONNX weights and run for minutes.
414
+ fails with `text array must be non-empty`. 0.2.2 embeds through the same
415
+ backend as the rest of AutoRAG (a remote `minSync.embedder`, else the
416
+ `minsync serve` daemon) instead of its own in-process model, and keeps vectors
417
+ in LanceDB. Switching from the old native model rebuilds the semantic index on
418
+ the next `index`. kakao needs `lazykatok` 0.3.3 or newer for the same reason.
403
419
  Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
404
420
  datasource is removed.
405
421
 
422
+ ### Browser history (Hister) and the shared embedding model
423
+
424
+ `hister` is the one datasource whose backing program AutoRAG does not install,
425
+ start, or bundle (it is AGPL-3.0-or-later and stays a separate program). Setting
426
+ it up is an agent task with these steps. Run each command; do not just describe it.
427
+
428
+ 1. **Probe first.** Add the datasource entry, then run `autorag setup --format json`
429
+ and read the `hister` object: `installed`, `running`, and `embedding`. Every
430
+ problem has a `remediation` string with the exact next command.
431
+ 2. **Install Hister** when `installed` is false: `brew install hister` (other
432
+ platforms: the release binaries at https://github.com/asciimoo/hister/releases).
433
+ 3. **Run it as a login item** when `running` is false: `brew services start hister`.
434
+ Hister must stay running: its browser extension only captures while it is up,
435
+ and a page visited while it is down is not re-sent. AutoRAG never starts it.
436
+ 4. **Install the browser extension** (Chrome Web Store "Hister", or Firefox
437
+ Add-ons). It must be done in the user's browser; tell the user if you cannot.
438
+ 5. **Share the embedding model** (below).
439
+ 6. **Before importing past history, install a skip list** in Hister's `rules.json`
440
+ (login, mail, documents, social and console domains, private hosts, token-bearing
441
+ URLs). `hister import browser history` re-downloads each URL *without* the
442
+ browser's cookies. See docs/datasource-skills.md, "Browser history via Hister".
443
+ 7. Add `"browser": { "type": "hister", "connector": { "semantic": true } }` under
444
+ `datasources`, then `autorag refresh` (only probes Hister) and
445
+ `autorag lite retrieve "<a question>" --scope '/browser/**' --json`.
446
+
447
+ **Shared embedding model.** Hister keeps its own vector store and embeds with
448
+ whatever its `config.yml` `semantic_search` block says. Pointing it at a different
449
+ model does not fail: vectors simply stop lining up and results get worse. So Hister
450
+ must use the same model as AutoRAG, which is the `minsync serve` daemon (model
451
+ `Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, with Qwen3's query instruction).
452
+
453
+ ```bash
454
+ autorag setup --format json # reports hister.embedding.differs: the keys that disagree
455
+ autorag setup --sync-hister # writes them, restarts Hister, re-embeds pages if needed
456
+ ```
457
+
458
+ - `--sync-hister` rewrites only `enable`, `embedding_endpoint`, `embedding_model`,
459
+ `dimensions`, `query_prefix`, `document_prefix` in Hister's config (a
460
+ `config.yml.bak.<time>` is kept next to it), then runs `brew services restart hister`,
461
+ waits for it to answer, and runs `hister reindex` when a change makes stored
462
+ vectors stale. Without the flag setup never writes Hister's config.
463
+ - The daemon port is chosen when `minsync serve` starts. After restarting the daemon,
464
+ run `autorag setup --sync-hister` again. It is idempotent: when nothing differs
465
+ it changes nothing and restarts nothing.
466
+ - It refuses, and says why, when the shared backend is a remote endpoint (browsing
467
+ text would leave the machine) or MinSync's in-process engine (Hister cannot reach
468
+ it; run `minsync serve --idle-timeout 0`, docs/minsync-setup.md).
469
+ - If `restartRequired` or `reindexRequired` is true in the report, the step was
470
+ not done (no Homebrew service, or it failed): run `brew services restart hister`
471
+ and `hister reindex` yourself. Check with `hister doctor`: the line
472
+ `server.index.embeddings` must be `ok`.
473
+ - A changed `query_prefix` alone needs a restart but no reindex. Never hand-edit the
474
+ `query_prefix` value across two lines: Hister's YAML parser folds the newline into
475
+ a space and the model's instruction is silently corrupted. `--sync-hister` writes
476
+ it correctly.
477
+
406
478
  ## Verify and build indexes
407
479
 
408
480
  Configuration alone is not a successful setup:
@@ -415,7 +487,8 @@ autorag refresh --json
415
487
  autorag search "summarize the collection" --top-k 3 --json --debug
416
488
  ```
417
489
 
418
- - `setup` probes runtime health, model profile, and datasource readiness.
490
+ - `setup` probes the embedding backend (`embedding.backend`), MinSync, and
491
+ datasource readiness.
419
492
  - `status` is model-free and path-opaque.
420
493
  - `health` resolves the single model, checks credential presence, and normally
421
494
  performs one live completion probe.
@@ -1,19 +0,0 @@
1
- import type { AgentTool } from "@earendil-works/pi-agent-core";
2
- import { Type } from "typebox";
3
- export declare const RECOMMEND_PEER_TARGETS_TOOL_NAME = "recommend_peer_targets";
4
- declare const recommendPeerTargetsSchema: Type.TObject<{
5
- query: Type.TString;
6
- }>;
7
- export interface PeerTargetRecommendation {
8
- readonly alias: string;
9
- readonly matchedTerms: readonly string[];
10
- readonly displayName?: string;
11
- readonly description?: string;
12
- }
13
- export interface RecommendPeerTargetsDetails {
14
- readonly method: "recommend_peer_targets";
15
- readonly resultCount: number;
16
- readonly matches: readonly PeerTargetRecommendation[];
17
- }
18
- export declare function createRecommendPeerTargetsTool(workspacePath: string): AgentTool<typeof recommendPeerTargetsSchema, RecommendPeerTargetsDetails>;
19
- export {};
@@ -1,51 +0,0 @@
1
- import type { AgentTool } from "@earendil-works/pi-agent-core";
2
- import { Type } from "typebox";
3
- import { type SimplexQueryState } from "../p2p/simplex-query-store.js";
4
- import type { SimplexTransport } from "../p2p/simplex-transport.js";
5
- import type { PeerQueryResponse } from "../p2p/wire.js";
6
- import { EvidenceLedger } from "./evidence-ledger.js";
7
- export declare const QUERY_PEER_AGENT_TOOL_NAME = "query_peer_agent";
8
- declare const queryPeerAgentSchema: Type.TObject<{
9
- alias: Type.TString;
10
- query: Type.TString;
11
- topK: Type.TOptional<Type.TInteger>;
12
- scope: Type.TOptional<Type.TString>;
13
- }>;
14
- export interface QueryPeerAgentResult {
15
- readonly number: number;
16
- readonly title: string;
17
- readonly summary: string;
18
- readonly source: string;
19
- readonly excerpt: string;
20
- }
21
- export interface QueryPeerAgentDetails {
22
- readonly method: "query_peer_agent";
23
- readonly ok: boolean;
24
- readonly alias: string;
25
- readonly contactId?: number;
26
- readonly pendingId?: string;
27
- readonly expiresAt?: string;
28
- readonly status?: "ok" | "rejected" | "pending";
29
- readonly answer?: string;
30
- readonly results?: readonly QueryPeerAgentResult[];
31
- readonly diagnostics?: PeerQueryResponse["diagnostics"];
32
- readonly fileCount?: number;
33
- readonly message?: string;
34
- }
35
- export interface QueryPeerAgentToolOptions {
36
- readonly workspacePath: string;
37
- readonly openTransport: () => Promise<SimplexTransport>;
38
- readonly fastTimeoutMs?: number;
39
- readonly autoStart?: boolean;
40
- readonly sessionId?: () => string | undefined;
41
- readonly onResponse?: (state: SimplexQueryState, response: PeerQueryResponse) => void;
42
- readonly onExpired?: (state: SimplexQueryState) => void;
43
- /** Run evidence ledger; each peer result is registered so the final emit can cite it by id. */
44
- readonly ledger?: EvidenceLedger;
45
- }
46
- export interface QueryPeerAgentTool extends AgentTool<typeof queryPeerAgentSchema, QueryPeerAgentDetails> {
47
- readonly ready: Promise<void>;
48
- close(): Promise<void>;
49
- }
50
- export declare function createQueryPeerAgentTool(options: QueryPeerAgentToolOptions): QueryPeerAgentTool;
51
- export {};
@@ -1,7 +0,0 @@
1
- import { type RuntimeStatus } from "../../embedding-runtime/index.js";
2
- import type { CommandContext } from "./types.js";
3
- export interface GatewayCommandDeps {
4
- runtimeStatus?: () => Promise<RuntimeStatus>;
5
- stopRuntime?: () => Promise<void>;
6
- }
7
- export declare function runGateway(ctx: CommandContext, deps?: GatewayCommandDeps): Promise<number>;
@@ -1,2 +0,0 @@
1
- import type { CommandContext } from "./types.js";
2
- export declare function runP2pPolicy(ctx: CommandContext): number;
@@ -1,7 +0,0 @@
1
- import type { CommandContext } from "./types.js";
2
- /** Resolve the workspace used for P2P identity/policy, matching `autorag serve`. */
3
- export declare function resolveP2pWorkspace(ctx: CommandContext): string;
4
- /**
5
- * `autorag p2p` — SimpleX peer trust management.
6
- */
7
- export declare function runP2p(ctx: CommandContext): Promise<number>;
@@ -1,19 +0,0 @@
1
- import type { InjectionClassifierModel } from "../../p2p/injection-classifier.js";
2
- import { type SimplexPeerServer, startSimplexPeerServer, syncSimplexPeers } from "../../p2p/simplex-server.js";
3
- import { type SimplexTransport, type StartSimplexOptions } from "../../p2p/simplex-transport.js";
4
- import { type CliConfig, type ResolveAgentModelOptions, type ResolvedAgentModel } from "../config.js";
5
- import type { CommandContext } from "./types.js";
6
- export interface ServeCommandDeps {
7
- readonly startSimplexPeerServer?: typeof startSimplexPeerServer;
8
- readonly startSimplexChat?: (options: StartSimplexOptions) => Promise<SimplexTransport>;
9
- readonly syncSimplexPeers?: typeof syncSimplexPeers;
10
- readonly waitUntilStopped?: (server: SimplexPeerServer) => Promise<void>;
11
- readonly modelResolver?: (config: CliConfig, options?: ResolveAgentModelOptions) => ResolvedAgentModel | Promise<ResolvedAgentModel>;
12
- }
13
- export declare function createInjectionClassifierModel(resolved: ResolvedAgentModel): InjectionClassifierModel;
14
- /**
15
- * `autorag serve` — start the P2P peer query server over SimpleX Chat.
16
- * Peers reach this installation through SimpleX; all security gates run
17
- * unchanged.
18
- */
19
- export declare function runServe(ctx: CommandContext, deps?: ServeCommandDeps): Promise<number>;
@@ -1,15 +0,0 @@
1
- import type { QmdEmbedResult, QmdEnsureResult, QmdOptions, QmdSearchMode, QmdSearchOptions, QmdSearchResult, QmdUpdateResult } from "./types.js";
2
- /**
3
- * Thin external `qmd` CLI wrapper for Obsidian vault indexing/search.
4
- * Every method spawns the `qmd` binary, never throws for expected failures,
5
- * and isolates config/cache under the AutoRAG workspace.
6
- */
7
- export declare class QmdClient {
8
- private readonly options;
9
- constructor(options?: QmdOptions);
10
- ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
11
- update(signal?: AbortSignal): Promise<QmdUpdateResult>;
12
- embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
13
- search(mode: QmdSearchMode, query: string, options?: QmdSearchOptions): Promise<QmdSearchResult>;
14
- private run;
15
- }
@@ -1,92 +0,0 @@
1
- import type { RetrievalOptions } from "../../../retrieval/types.js";
2
- /**
3
- * Search modes accepted by the external `qmd` CLI.
4
- *
5
- * - `search` — BM25 / FTS5 lexical search
6
- * - `vsearch` — dense vector semantic search
7
- */
8
- export type QmdSearchMode = "search" | "vsearch";
9
- /**
10
- * Configuration for {@link QmdClient}. The client spawns the `qmd` binary as a
11
- * child process and isolates config/cache under the AutoRAG workspace.
12
- */
13
- export interface QmdOptions {
14
- /** Explicit path to the `qmd` binary. Defaults to a bare `qmd` PATH lookup. */
15
- readonly binaryPath?: string;
16
- /** Spawn timeout in milliseconds. Default 60_000 (embed can be slow). */
17
- readonly timeoutMs?: number;
18
- /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
19
- readonly maxBufferBytes?: number;
20
- /** Absolute path to the Obsidian vault. Required for ensure/update. */
21
- readonly vaultPath?: string;
22
- /** AutoRAG workspace root used to isolate qmd config/cache. */
23
- readonly workspaceRoot?: string;
24
- /** Instance id used for source paths and collection naming. Default `default`. */
25
- readonly instanceId?: string;
26
- /** Explicit collection name; defaults to a sanitized instance id. */
27
- readonly collectionName?: string;
28
- /** Environment overrides merged on top of a restricted process env. */
29
- readonly env?: Readonly<Record<string, string | undefined>>;
30
- /** Explicit operator-owned qmd config directory, never rewritten. */
31
- readonly configPath?: string;
32
- }
33
- export declare const DEFAULT_QMD_BINARY = "qmd";
34
- export declare const DEFAULT_QMD_TIMEOUT_MS = 60000;
35
- export declare const DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
36
- export type QmdFailureReason = "binary-missing" | "not-configured" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
37
- export interface QmdFailure {
38
- readonly ok: false;
39
- readonly reason: QmdFailureReason;
40
- readonly stdout: string;
41
- readonly stderr: string;
42
- readonly code: number | null;
43
- }
44
- export interface QmdOk<T> {
45
- readonly ok: true;
46
- readonly data: T;
47
- readonly stdout: string;
48
- readonly stderr: string;
49
- readonly code: number;
50
- }
51
- export interface QmdUpdateInfo {
52
- readonly indexed: number;
53
- readonly updated: number;
54
- readonly unchanged: number;
55
- readonly removed: number;
56
- readonly needsEmbedding?: boolean;
57
- readonly metadata?: Readonly<Record<string, unknown>>;
58
- }
59
- export interface QmdEmbedInfo {
60
- readonly embedded: boolean;
61
- readonly metadata?: Readonly<Record<string, unknown>>;
62
- }
63
- export interface QmdEnsureInfo {
64
- readonly collectionName: string;
65
- readonly vaultPath: string;
66
- readonly configDir: string;
67
- }
68
- /** A scored search hit from `qmd search` / `qmd vsearch`. */
69
- export interface QmdSearchHit {
70
- readonly chunkId: string;
71
- readonly score: number;
72
- readonly content: string;
73
- readonly title?: string;
74
- readonly file?: string;
75
- readonly docid?: string;
76
- readonly metadata?: Readonly<Record<string, unknown>>;
77
- }
78
- export interface QmdSearchOk {
79
- readonly ok: true;
80
- readonly hits: readonly QmdSearchHit[];
81
- readonly stdout: string;
82
- readonly stderr: string;
83
- readonly code: number;
84
- readonly data: {
85
- readonly hits: readonly QmdSearchHit[];
86
- };
87
- }
88
- export type QmdEnsureResult = QmdOk<QmdEnsureInfo> | QmdFailure;
89
- export type QmdUpdateResult = QmdOk<QmdUpdateInfo> | QmdFailure;
90
- export type QmdEmbedResult = QmdOk<QmdEmbedInfo> | QmdFailure;
91
- export type QmdSearchResult = QmdSearchOk | QmdFailure;
92
- export type QmdSearchOptions = RetrievalOptions;
@@ -1,30 +0,0 @@
1
- export { CacheError } from "./types.js";
2
- export interface CacheAsset {
3
- readonly id: string;
4
- readonly url: string;
5
- readonly filename: string;
6
- readonly sha256: string;
7
- readonly kind?: "model" | "runtime";
8
- readonly archiveMembers?: readonly string[];
9
- }
10
- export declare function cacheDirectory(kind: "models" | "runtime", cacheRoot?: string): string;
11
- export interface CacheOptions {
12
- readonly cacheRoot?: string;
13
- readonly offline?: boolean;
14
- readonly fetch?: typeof globalThis.fetch;
15
- /** Total budget (ms) for a single asset download. Defaults to 10 minutes so large models can transfer. */
16
- readonly downloadTimeoutMs?: number;
17
- }
18
- export declare function verifyCacheEntry(path: string, expectedSha256: string): Promise<string>;
19
- /**
20
- * Marker a distributor writes inside `<archive>.extracted/` when it ships the
21
- * runtime already extracted instead of as the pinned archive. Its content is the
22
- * archive's SHA-256, so a tree provisioned for another runtime build is never
23
- * taken for this one. A signed and notarized macOS app needs this: every Mach-O
24
- * it carries must be re-signed with the app's Developer ID, so the binaries can
25
- * no longer be packed back into an archive that matches the pinned digest.
26
- */
27
- export declare const PROVISIONED_RUNTIME_MARKER = ".autorag-archive-sha256";
28
- export declare function hasVerifiedAsset(asset: CacheAsset, cacheRoot?: string): Promise<boolean>;
29
- export declare function downloadAsset(asset: CacheAsset, options?: CacheOptions): Promise<string>;
30
- export declare function importAsset(sourcePath: string, asset: CacheAsset, options?: CacheOptions): Promise<string>;
@@ -1,41 +0,0 @@
1
- import type { ProfileId } from "./types.js";
2
- export interface EmbeddingIdentity {
3
- readonly provider: string;
4
- readonly model: string;
5
- readonly dimension: number;
6
- }
7
- /** Embedder contract: identity is async because the gateway resolves it lazily. */
8
- export interface Embedder {
9
- identity(): Promise<EmbeddingIdentity>;
10
- embed(texts: readonly string[]): Promise<readonly (readonly number[])[]>;
11
- }
12
- export interface GatewayEmbedderOptions {
13
- /** Runtime resolver; defaults to the shared embedding-runtime ensureRuntime. */
14
- readonly runtime?: {
15
- ensureRuntime(input?: {
16
- profileId?: ProfileId;
17
- cachedOnly?: boolean;
18
- }): Promise<{
19
- baseUrl: string;
20
- identity: {
21
- provider: string;
22
- model: string;
23
- dimension: number;
24
- };
25
- }>;
26
- };
27
- readonly profileId?: ProfileId;
28
- readonly fetchImpl?: typeof fetch;
29
- }
30
- /**
31
- * Texts per `/v1/embeddings` call. The llama.cpp upstream rejects very large
32
- * batches (HTTP 413 on a full-archive sync), so embedding fans out in
33
- * bounded sequential batches while preserving input order.
34
- */
35
- export declare const EMBED_BATCH_SIZE = 32;
36
- /**
37
- * Production embedder backed by the loopback autorag-gateway. The runtime is
38
- * ensured lazily with `cachedOnly` so embedding never triggers a model
39
- * download; an unprimed cache surfaces as a caller-side degrade.
40
- */
41
- export declare function createGatewayEmbedder(options?: GatewayEmbedderOptions): Embedder;
@@ -1,27 +0,0 @@
1
- import type { RuntimeProfile } from "./types.js";
2
- export type GatewayErrorCode = "non-loopback-host" | "non-loopback-upstream" | "bad-request" | "batch-limit" | "upstream" | "upstream-violation" | "timeout";
3
- export declare class GatewayError extends Error {
4
- readonly code: GatewayErrorCode;
5
- readonly status: number;
6
- constructor(code: GatewayErrorCode, message: string, status?: number, options?: {
7
- readonly cause?: unknown;
8
- });
9
- }
10
- export interface StartEmbeddingGatewayOptions {
11
- readonly profile: RuntimeProfile;
12
- readonly host?: string;
13
- readonly port?: number;
14
- readonly upstreamUrl: string;
15
- readonly fetch?: typeof fetch;
16
- readonly maxInputs?: number;
17
- readonly maxCharacters?: number;
18
- readonly timeoutMs?: number;
19
- }
20
- export interface EmbeddingGateway {
21
- readonly host: string;
22
- readonly port: number;
23
- readonly url: string;
24
- close(): Promise<void>;
25
- }
26
- export declare function isLoopbackHost(host: string): boolean;
27
- export declare function startEmbeddingGateway(options: StartEmbeddingGatewayOptions): Promise<EmbeddingGateway>;
@@ -1,120 +0,0 @@
1
- import { type CacheOptions } from "./cache.js";
2
- import { startEmbeddingGateway } from "./gateway.js";
3
- import { type SupervisorStatus } from "./supervisor.js";
4
- import type { BackendKind, HealthStatus, ProfileId, RuntimeProfile } from "./types.js";
5
- export interface EmbeddingRuntimeCache {
6
- downloadAsset(asset: {
7
- id: string;
8
- url: string;
9
- filename: string;
10
- sha256: string;
11
- kind?: "model" | "runtime";
12
- archiveMembers?: readonly string[];
13
- }, options?: CacheOptions): Promise<string>;
14
- importAsset(sourcePath: string, asset: {
15
- id: string;
16
- url: string;
17
- filename: string;
18
- sha256: string;
19
- kind?: "model" | "runtime";
20
- archiveMembers?: readonly string[];
21
- }, options?: CacheOptions): Promise<string>;
22
- verifyCacheEntry(path: string, expectedSha256: string): Promise<string>;
23
- hasVerifiedAsset?(asset: {
24
- id: string;
25
- url: string;
26
- filename: string;
27
- sha256: string;
28
- kind?: "model" | "runtime";
29
- archiveMembers?: readonly string[];
30
- }, cacheRoot?: string): Promise<boolean>;
31
- }
32
- export interface EmbeddingRuntimeSupervisor {
33
- ensureRunning(): Promise<SupervisorStatus>;
34
- shutdown(): Promise<void>;
35
- status(): SupervisorStatus;
36
- }
37
- export interface EmbeddingRuntimeGateway {
38
- readonly url: string;
39
- close(): Promise<void>;
40
- }
41
- export interface EmbeddingRuntimeOptions {
42
- cacheRoot?: string;
43
- backend?: BackendKind;
44
- offline?: boolean;
45
- platform?: "darwin-arm64-metal" | "win-x64-cpu" | "win-x64-vulkan";
46
- fetch?: typeof fetch;
47
- supervisor?: EmbeddingRuntimeSupervisor;
48
- cache?: EmbeddingRuntimeCache;
49
- gatewayFactory?: (options: Parameters<typeof startEmbeddingGateway>[0]) => Promise<EmbeddingRuntimeGateway>;
50
- }
51
- export interface EnsureRuntimeOptions extends EmbeddingRuntimeOptions {
52
- profileId?: ProfileId;
53
- /** Refuse downloads; used by MinSync refresh/query paths. */
54
- cachedOnly?: boolean;
55
- }
56
- export interface RuntimeIdentity {
57
- readonly profileId: ProfileId;
58
- readonly provider: string;
59
- readonly model: string;
60
- readonly dimension: number;
61
- readonly runtimeBuild: string;
62
- readonly modelRevision: string;
63
- readonly artifactSha256: string;
64
- }
65
- export interface EnsuredRuntime {
66
- readonly baseUrl: string;
67
- readonly profile: RuntimeProfile;
68
- readonly identity: RuntimeIdentity;
69
- readonly supervisor: SupervisorStatus;
70
- }
71
- export interface RuntimeStatus {
72
- readonly state: SupervisorStatus["state"];
73
- readonly backend: BackendKind;
74
- readonly model: string;
75
- readonly pid?: number;
76
- readonly port?: number;
77
- readonly uptimeMs?: number;
78
- readonly lastError?: string;
79
- readonly profileId?: ProfileId;
80
- readonly baseUrl?: string;
81
- readonly health: HealthStatus;
82
- }
83
- export declare function createEmbeddingRuntime(options?: EmbeddingRuntimeOptions): {
84
- resolveRuntimeProfile: (profileId?: ProfileId) => RuntimeProfile;
85
- ensureRuntime: (input?: EnsureRuntimeOptions) => Promise<EnsuredRuntime>;
86
- releaseRuntimeHandles: () => Promise<void>;
87
- stopRuntime: () => Promise<void>;
88
- runtimeStatus: () => Promise<RuntimeStatus>;
89
- prefetchModel: (profileId?: ProfileId) => Promise<{
90
- profileId: "qwen3-embedding-0.6b";
91
- path: string;
92
- }>;
93
- importModel: (profileId: ProfileId | undefined, sourcePath: string) => Promise<{
94
- profileId: "qwen3-embedding-0.6b";
95
- path: string;
96
- }>;
97
- verifyModel: (profileId?: ProfileId) => Promise<{
98
- profileId: "qwen3-embedding-0.6b";
99
- path: string;
100
- hash: string;
101
- }>;
102
- };
103
- export declare const resolveRuntimeProfile: (profileId?: ProfileId) => RuntimeProfile;
104
- export declare const ensureRuntime: (input?: EnsureRuntimeOptions) => Promise<EnsuredRuntime>;
105
- export declare const releaseRuntimeHandles: () => Promise<void>;
106
- export declare const stopRuntime: () => Promise<void>;
107
- export declare const runtimeStatus: () => Promise<RuntimeStatus>;
108
- export declare const prefetchModel: (profileId?: ProfileId) => Promise<{
109
- profileId: "qwen3-embedding-0.6b";
110
- path: string;
111
- }>;
112
- export declare const importModel: (profileId: ProfileId | undefined, sourcePath: string) => Promise<{
113
- profileId: "qwen3-embedding-0.6b";
114
- path: string;
115
- }>;
116
- export declare const verifyModel: (profileId?: ProfileId) => Promise<{
117
- profileId: "qwen3-embedding-0.6b";
118
- path: string;
119
- hash: string;
120
- }>;
@@ -1,20 +0,0 @@
1
- import type { BackendKind, PlatformId, ProfileId, RuntimeAsset, RuntimeProfile } from "./types.js";
2
- export declare function resolveProfile(profileId: ProfileId): RuntimeProfile;
3
- export declare function platformRuntimeAsset(platform: PlatformId): RuntimeAsset;
4
- export declare function selectPlatformAsset(platform: PlatformId, backend: BackendKind, vulkanAvailable?: boolean): RuntimeAsset;
5
- export declare const MODEL_ASSETS: {
6
- readonly qwen3: {
7
- readonly url: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf";
8
- readonly revision: "370f27d7550e0def9b39c1f16d3fbaa13aa67728";
9
- readonly sha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439";
10
- readonly license: "Apache-2.0";
11
- readonly noticeReference: "Qwen3 model card and Apache-2.0 license";
12
- readonly intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.";
13
- readonly unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.";
14
- readonly upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B";
15
- readonly modelCard: "docs/model-cards/qwen3-embedding-0.6b.md";
16
- };
17
- };
18
- export declare const MODEL_ASSET_URLS: {
19
- readonly qwen3: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf";
20
- };