@autorag/librarian 2.10.5 → 2.10.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/NOTICE +6 -9
  2. package/README.md +19 -18
  3. package/dist/agent/agent.d.ts +45 -48
  4. package/dist/agent/context-token-guard.d.ts +11 -0
  5. package/dist/agent/discord-remote.d.ts +30 -0
  6. package/dist/agent/embedding-controls.d.ts +22 -0
  7. package/dist/agent/index.d.ts +1 -2
  8. package/dist/agent/pi-session.d.ts +0 -1
  9. package/dist/agent/query-decomposition.d.ts +1 -1
  10. package/dist/agent/search-documents.d.ts +2 -2
  11. package/dist/agent/system-prompt.d.ts +1 -1
  12. package/dist/cli/commands/models.d.ts +0 -14
  13. package/dist/cli/commands/search.d.ts +3 -4
  14. package/dist/cli/config.d.ts +7 -19
  15. package/dist/cli/index.d.ts +1 -1
  16. package/dist/cli/index.js +3134 -6671
  17. package/dist/cli/setup.d.ts +13 -36
  18. package/dist/cloud/config.d.ts +3 -3
  19. package/dist/core.js +5144 -7954
  20. package/dist/datasource/embedding-env.d.ts +29 -0
  21. package/dist/datasource/sidecar/config.d.ts +3 -0
  22. package/dist/datasource/sidecar/index.d.ts +1 -1
  23. package/dist/datasource/skills/discrawl/client.d.ts +1 -0
  24. package/dist/datasource/skills/discrawl/embedding.d.ts +17 -0
  25. package/dist/datasource/skills/discrawl/types.d.ts +11 -3
  26. package/dist/datasource/skills/factory.d.ts +14 -1
  27. package/dist/datasource/skills/github-gist/semantic.d.ts +5 -5
  28. package/dist/datasource/skills/github-gist/skill.d.ts +2 -3
  29. package/dist/datasource/skills/lazykatok/types.d.ts +9 -1
  30. package/dist/datasource/skills/mailcrawl/types.d.ts +9 -1
  31. package/dist/embedding/daemon.d.ts +57 -0
  32. package/dist/embedding/embedder.d.ts +59 -0
  33. package/dist/embedding/http-embedder.d.ts +28 -0
  34. package/dist/embedding/in-process.d.ts +19 -0
  35. package/dist/embedding/index.d.ts +28 -0
  36. package/dist/embedding/managed.d.ts +44 -0
  37. package/dist/embedding/prefixes.d.ts +18 -0
  38. package/dist/embedding/settings.d.ts +9 -0
  39. package/dist/index.js +5153 -7965
  40. package/dist/mcp/index.js +4890 -7700
  41. package/dist/memory/context.d.ts +1 -1
  42. package/dist/memory/similar-queries.d.ts +1 -1
  43. package/dist/minsync/binary.d.ts +21 -0
  44. package/dist/minsync/client.d.ts +11 -10
  45. package/dist/minsync/index.d.ts +1 -1
  46. package/dist/minsync/installer.d.ts +1 -1
  47. package/dist/minsync/method.d.ts +3 -11
  48. package/dist/minsync/process.d.ts +1 -0
  49. package/dist/minsync/types.d.ts +0 -2
  50. package/dist/retrieval/rerank.d.ts +10 -3
  51. package/package.json +3 -3
  52. package/skills/autorag/SKILL.md +3 -8
  53. package/skills/autorag-doctor/SKILL.md +46 -26
  54. package/skills/autorag-lite-setup/SKILL.md +8 -8
  55. package/skills/autorag-setup/SKILL.md +39 -23
  56. package/dist/agent/peer-target-tool.d.ts +0 -19
  57. package/dist/agent/query-peer-tool.d.ts +0 -51
  58. package/dist/cli/commands/gateway.d.ts +0 -7
  59. package/dist/cli/commands/p2p-policy.d.ts +0 -2
  60. package/dist/cli/commands/p2p.d.ts +0 -7
  61. package/dist/cli/commands/serve.d.ts +0 -19
  62. package/dist/embedding-runtime/cache.d.ts +0 -30
  63. package/dist/embedding-runtime/gateway-embedder.d.ts +0 -41
  64. package/dist/embedding-runtime/gateway.d.ts +0 -27
  65. package/dist/embedding-runtime/index.d.ts +0 -120
  66. package/dist/embedding-runtime/manifest.d.ts +0 -20
  67. package/dist/embedding-runtime/supervisor.d.ts +0 -74
  68. package/dist/embedding-runtime/types.d.ts +0 -70
  69. package/dist/p2p/approval-store.d.ts +0 -32
  70. package/dist/p2p/egress-gate.d.ts +0 -22
  71. package/dist/p2p/file-sharing.d.ts +0 -33
  72. package/dist/p2p/injection-classifier.d.ts +0 -33
  73. package/dist/p2p/injection-gate.d.ts +0 -37
  74. package/dist/p2p/pii-gate.d.ts +0 -47
  75. package/dist/p2p/policy-filter.d.ts +0 -15
  76. package/dist/p2p/policy.d.ts +0 -77
  77. package/dist/p2p/simplex-query-store.d.ts +0 -44
  78. package/dist/p2p/simplex-server.d.ts +0 -115
  79. package/dist/p2p/simplex-transport.d.ts +0 -62
  80. package/dist/p2p/wire.d.ts +0 -39
  81. package/licenses/embedding-assets.json +0 -85
  82. package/licenses/llama.cpp-MIT.txt +0 -21
  83. package/licenses/qwen3-embedding-notice.txt +0 -13
@@ -1,4 +1,4 @@
1
- import type { Embedder } from "../embedding-runtime/gateway-embedder.js";
1
+ import type { Embedder } from "../embedding/embedder.js";
2
2
  import type { JudgedEvidenceRecord } from "./judged-evidence.js";
3
3
  import type { RetrievalInsight, RetrievalMemory } from "./memory.js";
4
4
  import { type SimilarQuestion } from "./similar-queries.js";
@@ -1,4 +1,4 @@
1
- import type { Embedder } from "../embedding-runtime/gateway-embedder.js";
1
+ import type { Embedder } from "../embedding/embedder.js";
2
2
  import type { JudgedEvidenceRecord } from "./judged-evidence.js";
3
3
  /** How many similar past questions memory brings back for one question. */
4
4
  export declare const SIMILAR_QUESTION_LIMIT = 25;
@@ -0,0 +1,21 @@
1
+ import { type EnsureMinSyncBinaryOptions } from "./installer.js";
2
+ export interface ResolveMinSyncBinaryOptions {
3
+ /** Workspace project root; the cached binary lives under `<root>/.autorag/bin`. */
4
+ readonly root: string;
5
+ /** Test/embedded override; CLI config intentionally does not persist this. */
6
+ readonly binaryPath?: string;
7
+ readonly autoInstall?: boolean;
8
+ readonly installer?: Omit<EnsureMinSyncBinaryOptions, "root">;
9
+ /** False for query paths: they use an existing binary or fail, and never wait on `cargo install`. */
10
+ readonly install?: boolean;
11
+ }
12
+ /** Resolve the `minsync` executable from PATH directories. Returns the first match or undefined. */
13
+ export declare function lookupMinSyncInPath(env: NodeJS.ProcessEnv): string | undefined;
14
+ /**
15
+ * Resolve the MinSync CLI from the user's global PATH first, then the workspace cache, and finally the
16
+ * verified auto-install fallback. Throws {@link MinSyncRequiredError}: MinSync is required, so there is
17
+ * no "no binary" value to degrade on.
18
+ */
19
+ export declare function resolveMinSyncBinary(options: ResolveMinSyncBinaryOptions): Promise<string>;
20
+ /** True when no `minsync` is resolvable right now without installing one. */
21
+ export declare function isMinSyncBinaryMissing(options: Pick<ResolveMinSyncBinaryOptions, "root" | "binaryPath">): boolean;
@@ -1,18 +1,9 @@
1
- import type { EnsuredRuntime } from "../embedding-runtime/index.js";
2
1
  import type { MinSyncEmbedderConfig, MinSyncQueryHit, MinSyncSyncResult } from "./types.js";
3
- export declare const MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.";
4
- export interface MinSyncRuntime {
5
- ensureRuntime(options?: {
6
- readonly profileId?: MinSyncEmbedderConfig["profile"];
7
- readonly cachedOnly?: boolean;
8
- }): Promise<EnsuredRuntime>;
9
- }
10
2
  export interface MinSyncClientOptions {
11
3
  readonly binaryPath: string;
12
4
  readonly workspacePath: string;
13
5
  readonly embedder?: MinSyncEmbedderConfig;
14
6
  readonly maxChunkSize?: number;
15
- readonly runtime?: MinSyncRuntime;
16
7
  }
17
8
  /** MinSync's native query modes: vector, BM25, and hybrid. */
18
9
  export type MinSyncQueryMode = "vector" | "bm25" | "hybrid";
@@ -31,8 +22,12 @@ export declare class MinSyncClient {
31
22
  private readonly workspacePath;
32
23
  private readonly embedder;
33
24
  private readonly maxChunkSize;
34
- private readonly runtime;
35
25
  constructor(options: MinSyncClientOptions);
26
+ /**
27
+ * What MinSync is told to embed with. No `id`, or a `native:` id, is MinSync's own Qwen3 engine, which
28
+ * MinSync provisions and runs (in-process, or through a `minsync serve` daemon it finds on its own).
29
+ * Anything else is an endpoint the operator configured, passed through untouched.
30
+ */
36
31
  private effectiveEmbedder;
37
32
  private readIdentity;
38
33
  private writeIdentity;
@@ -50,4 +45,10 @@ export declare class MinSyncClient {
50
45
  private checkAndSync;
51
46
  query(text: string, topK: number, mode?: MinSyncQueryMode, signal?: AbortSignal): Promise<readonly MinSyncQueryHit[]>;
52
47
  private spawn;
48
+ /**
49
+ * MinSync's `openai:` embedder reads its bearer token from `OPENAI_API_KEY` only. The operator names the
50
+ * variable that holds their key with `apiKeyEnv`, so that value is what the child receives as
51
+ * `OPENAI_API_KEY`. A local `native:` embedder needs no key and the environment is left as it is.
52
+ */
53
+ private childEnv;
53
54
  }
@@ -1,4 +1,4 @@
1
- export { MINSYNC_OLLAMA_MIGRATION_MESSAGE, MinSyncClient, type MinSyncClientOptions, MinSyncQueryError, type MinSyncQueryMode, } from "./client.js";
1
+ export { MinSyncClient, type MinSyncClientOptions, MinSyncQueryError, type MinSyncQueryMode, } from "./client.js";
2
2
  export { configuredVectorDimension, DEFAULT_MINSYNC_EMBEDDER_CONFIG, DEFAULT_MINSYNC_EMBEDDER_DIMENSION, DEFAULT_MINSYNC_EMBEDDER_ID, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE, type MinSyncEmbeddingIdentity, minSyncConfigPath, minSyncEmbeddingIdentityPath, rewriteEmbedderConfig, } from "./embedder-config.js";
3
3
  export { MinSyncRequiredError } from "./errors.js";
4
4
  export { type EnsureMinSyncBinaryOptions, ensureMinSyncBinary, executableName, fetchLatestMinSyncRelease, type InstalledMinSyncBinary, MINSYNC_VERSION, type MinSyncRelease, type MinSyncReleaseAsset, MinSyncReleaseError, selectReleaseAsset, } from "./installer.js";
@@ -1,4 +1,4 @@
1
- export declare const MINSYNC_VERSION = "0.4.10";
1
+ export declare const MINSYNC_VERSION = "0.4.11";
2
2
  export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
3
3
  export interface MinSyncReleaseAsset {
4
4
  readonly name: string;
@@ -1,7 +1,6 @@
1
1
  import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
2
2
  import type { MinSyncQueryMode } from "./client.js";
3
- import { type MinSyncRuntime } from "./client.js";
4
- import { type EnsureMinSyncBinaryOptions } from "./installer.js";
3
+ import type { EnsureMinSyncBinaryOptions } from "./installer.js";
5
4
  import type { MinSyncEmbedderConfig, MinSyncSyncResult } from "./types.js";
6
5
  export interface MinSyncVectorMethodOptions {
7
6
  readonly root: string;
@@ -13,7 +12,6 @@ export interface MinSyncVectorMethodOptions {
13
12
  readonly embedder?: MinSyncEmbedderConfig;
14
13
  readonly maxChunkSize?: number;
15
14
  readonly mode?: MinSyncQueryMode;
16
- readonly runtime?: MinSyncRuntime;
17
15
  /** Default `topK` used when the caller omits one. Default 50. */
18
16
  readonly defaultTopK?: number;
19
17
  /** Fetch cap applied when a scope narrows the query. Default 100. */
@@ -29,7 +27,6 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
29
27
  private readonly embedder;
30
28
  private readonly maxChunkSize;
31
29
  private readonly mode;
32
- private readonly runtime;
33
30
  private readonly defaultTopK;
34
31
  private readonly scopedTopK;
35
32
  constructor(options: MinSyncVectorMethodOptions);
@@ -45,13 +42,8 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
45
42
  retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
46
43
  private retrieveUnlocked;
47
44
  /**
48
- * Resolve the MinSync CLI from the user's global PATH first, then the
49
- * workspace cache, and finally the verified auto-install fallback. The
50
- * auto-install runs only for `sync` (refresh); queries pass
51
- * `install: false` so a user's turn never waits on `cargo install`.
52
- *
53
- * Returns the executable path, or throws {@link MinSyncRequiredError}:
54
- * MinSync is required, so there is no "no binary" value to degrade on.
45
+ * Queries pass `install: false` so a user's turn never waits on `cargo install`; only `sync` (refresh)
46
+ * may install. See {@link resolveMinSyncBinary}.
55
47
  */
56
48
  protected resolveBinary(options?: {
57
49
  readonly install?: boolean;
@@ -8,4 +8,5 @@ export interface ProcessResult {
8
8
  export declare function spawnProcess(command: string, args: readonly string[], cwd: string, options?: {
9
9
  readonly timeoutMs?: number;
10
10
  readonly signal?: AbortSignal;
11
+ readonly env?: NodeJS.ProcessEnv;
11
12
  }): Promise<ProcessResult>;
@@ -1,8 +1,6 @@
1
1
  /** Embedder configuration for MinSync vector indexing. Shared by config and method layers. */
2
2
  import type { MinSyncLexicalLanguage } from "./lexical-language.js";
3
3
  export interface MinSyncEmbedderConfig {
4
- /** Select an AutoRAG-owned embedding runtime profile. */
5
- readonly profile?: "qwen3-embedding-0.6b";
6
4
  readonly id?: string;
7
5
  readonly baseUrl?: string;
8
6
  /** Environment variable name whose value holds the embedder API key. /^[A-Za-z_][A-Za-z0-9_]*$/ */
@@ -24,6 +24,13 @@ export declare const DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY";
24
24
  * chat LLMs, so the project's 2026+ model rule does not cover them.
25
25
  */
26
26
  export declare const DEFAULT_DAZZI_RERANK_MODEL = "qwen/qwen3-reranker-8b";
27
+ /**
28
+ * Default per-request rerank timeout. A reranker that never answers would stall
29
+ * the fast answer. Measured against the hosted reranker, a 172-document pool
30
+ * takes ~15s and a 300-document pool ~45s, so 60s clears the largest baseline
31
+ * pool (3 x 100 candidates) with headroom.
32
+ */
33
+ export declare const DEFAULT_RERANK_TIMEOUT_MS = 60000;
27
34
  /** Default number of merged chunks kept after reranking. */
28
35
  export declare const DEFAULT_RERANK_TOP_N = 25;
29
36
  /** Rerank providers {@link createReranker} can build. */
@@ -65,7 +72,7 @@ export interface Reranker {
65
72
  */
66
73
  rerank(query: string, results: readonly RetrievalResult[], options?: RerankOptions): Promise<RetrievalResult[]>;
67
74
  }
68
- /** Minimal rerank client shape, so tests can inject a fake without the SDK. */
75
+ /** Minimal rerank client shape, so tests can inject a fake without network access. */
69
76
  export interface RerankClient {
70
77
  rerank(request: {
71
78
  requestBody: {
@@ -78,7 +85,7 @@ export interface RerankClient {
78
85
  signal?: AbortSignal;
79
86
  }): Promise<RerankResponse>;
80
87
  }
81
- /** Response body returned by OpenRouter's rerank router. */
88
+ /** Rerank response as {@link OpenRouterReranker} consumes it. */
82
89
  export interface RerankResponse {
83
90
  readonly results: ReadonlyArray<{
84
91
  readonly index: number;
@@ -123,7 +130,7 @@ export interface CreateRerankerOptions extends OpenRouterRerankerOptions {
123
130
  */
124
131
  export declare function createReranker(config: CreateRerankerOptions | false | undefined): Reranker | undefined;
125
132
  /**
126
- * Reranker backed by an OpenRouter-compatible rerank router.
133
+ * Reranker backed by an OpenRouter-compatible rerank endpoint.
127
134
  *
128
135
  * OpenRouter is the default provider; the hosted Dazzi Models plan serves the
129
136
  * same wire shape at `<root>/v1/rerank`, so {@link createReranker} reuses this
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@autorag/librarian",
3
- "version": "2.10.5",
3
+ "version": "2.10.6",
4
4
  "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -44,7 +44,6 @@
44
44
  "check": "biome check --write .",
45
45
  "lint": "biome check .",
46
46
  "typecheck": "tsc --noEmit",
47
- "check:embedding-manifest": "node scripts/check-embedding-manifest.mjs",
48
47
  "supply-chain": "bun scripts/supply-chain/evaluate.ts gate --sbom sbom.local.cdx.json"
49
48
  },
50
49
  "engines": {
@@ -57,8 +56,8 @@
57
56
  "@earendil-works/pi-tui": "1.2.0",
58
57
  "@llblab/pi-telegram": "0.57.2",
59
58
  "@modelcontextprotocol/server": "^2.3.1",
60
- "@openrouter/sdk": "^1.4.25",
61
59
  "chardet": "^2.2.0",
60
+ "discord.js": "14.27.0",
62
61
  "fast-xml-parser": "^5.11.2",
63
62
  "iconv-lite": "^0.7.3",
64
63
  "jev-use": "^0.8.0",
@@ -66,6 +65,7 @@
66
65
  "kordoc": "^4.21.30",
67
66
  "mailparser": "^3.9.38",
68
67
  "pdfjs-dist": "^6.4.299",
68
+ "pi-discord-remote": "0.2.4",
69
69
  "puppeteer-core": "^25.13.0",
70
70
  "slides-grab": "^1.5.2",
71
71
  "smol-toml": "1.9.1",
@@ -142,8 +142,6 @@ autorag index rebuild --yes --json
142
142
  autorag index reset --method parsed --yes --json
143
143
  autorag memory inspect --json
144
144
  autorag tui
145
- autorag serve --force
146
- autorag p2p policy list --json
147
145
  ```
148
146
 
149
147
  Prefer a full refresh so parsed mirrors, MinSync, Jikji, configured
@@ -153,9 +151,7 @@ aligned. `--method` accepts
153
151
  a `--method` name. Use `--method` only for deliberate narrowing. Scheduled
154
152
  maintenance should use non-daemon `autorag watch --once`, typically every
155
153
  1 hour, with the same config used by search and no overlapping runs.
156
- `autorag tui` is the shipped beta terminal UI. `autorag serve` / `autorag p2p`
157
- are opt-in SimpleX peer sharing (disabled until `p2p.enabled` is true, unless
158
- `--force`).
154
+ `autorag tui` is the shipped beta terminal UI.
159
155
 
160
156
  Reset and rebuild commands remove only selected workspace `.autorag` indexes.
161
157
  Never target source documents. `memory inspect` is read-only and path-opaque.
@@ -169,8 +165,7 @@ Never target source documents. `memory inspect` is read-only and path-opaque.
169
165
  - Preserve real source mapping.
170
166
  - Prefer `--json --debug` when another agent consumes search output.
171
167
  - Do not invent CLI commands. `autorag --help` is the command list of record,
172
- including the shipped `setup`, `gateway`, `models`, `serve`, `p2p`, `tui`, and `lite`
168
+ including the shipped `setup`, `models`, `tui`, and `lite`
173
169
  commands. `autorag <command> --help` prints that command's own flags, and
174
170
  `autorag --version` prints the installed version. See
175
- `docs/embedding-runtime.md` for the local embedding runtime
176
- and gateway setup.
171
+ `docs/minsync-setup.md#embedding-backends` for MinSync setup and embedding-backend selection.
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: autorag-doctor
3
- description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding gateway, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the gateway will not start, or the user asks to check, diagnose, verify, or repair AutoRAG.
3
+ description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
4
4
  license: Apache-2.0
5
5
  ---
6
6
 
@@ -29,7 +29,7 @@ Always finish with the status table in [Report](#report).
29
29
  ```bash
30
30
  autorag status --json
31
31
  autorag health --json
32
- autorag gateway status --format json
32
+ autorag setup --format json
33
33
  ```
34
34
 
35
35
  - `status` reports `state`, `stale`, `diagnostics`, and per-component state for
@@ -38,8 +38,12 @@ autorag gateway status --format json
38
38
  - `health` resolves the single search model and does one live completion probe.
39
39
  `--skip-probes` only proves config shape, never live access; do not claim a
40
40
  healthy model from it.
41
- - The gateway is on-demand: `stopped` is normal when nothing is embedding.
42
- `unavailable` **while** a refresh is running is a real failure.
41
+ - `setup --format json` reports `embedding.backend` (`remote`, `daemon`, or
42
+ `in-process`) and the resolved `provider`/`model`/`dimension` when a backend
43
+ is alive, or `embedding.reason` and a `remediation` when none is. It only
44
+ probes; it never starts a daemon. Semantic search needs the daemon (or a
45
+ remote endpoint) to be running: a backend that is absent is a real
46
+ `embedding-unavailable` state, not normal.
43
47
 
44
48
  Config lives at `--config`, `AUTORAG_CONFIG`, `$AUTORAG_HOME/config.json`, or
45
49
  `~/.autorag/config.json`. Read `searchPaths`, `workspacePath`, `minSync`,
@@ -129,19 +133,30 @@ autorag search "summarize the collection" --top-k 3 --json --debug
129
133
 
130
134
  ## 4. Repair playbook
131
135
 
132
- ### Orphan lock
136
+ ### Stuck embedding daemon
133
137
 
134
- The embedding runtime keeps `embedding-runtime.lock`, `embedding-runtime.pid`,
135
- and `embedding-runtime.port` in `AUTORAG_HOME`. It reclaims them automatically
136
- when the recorded PID is dead; a `lock-conflict` means the PID is **alive**.
138
+ AutoRAG owns no lock or pid file for embeddings. A running `minsync serve`
139
+ publishes `host`, `port`, and `pid` under `~/.cache/minsync/serve/<instance>/`
140
+ and removes them on a clean stop. A leftover file from a crashed daemon is
141
+ harmless: AutoRAG only adopts a daemon whose `GET /health` answers as a loopback
142
+ server of the wanted model, so a stale entry is skipped.
137
143
 
138
144
  ```bash
139
- autorag gateway status --format json
140
- autorag gateway stop
145
+ ls ~/.cache/minsync/serve/* 2>/dev/null
146
+ autorag setup --format json
141
147
  ```
142
148
 
143
- Only when `gateway stop` cannot clear it, and the recorded PID is confirmed
144
- dead (`kill -0 PID` fails), remove the three files by hand and retry.
149
+ To diagnose an embedding backend that is not alive, `setup --format json` and
150
+ `status` say so (`embedding.reason`, and the `embedding-unavailable`
151
+ diagnostic). AutoRAG never starts the daemon: run `minsync serve
152
+ --idle-timeout 0` by hand (or as a launchd/systemd service, see
153
+ `docs/minsync-setup.md`) and read its own error, or press Ctrl+Shift+E in the
154
+ TUI. When no daemon answers and the in-process addon
155
+ (`@nomadamas/minsync-native`) is missing, that is the `embedder-unavailable`
156
+ state. The daemon stops itself after `--idle-timeout` seconds (default 300), so
157
+ a semantic failure after a quiet spell usually means it timed out. Stop an
158
+ orphaned daemon by the pid from `~/.cache/minsync/serve/<instance>/pid` after
159
+ confirming no live refresh holds it.
145
160
 
146
161
  Index locks are owned by their engines: MinSync/tantivy locks under
147
162
  `<workspace>/.autorag/`, and per-CLI locks such as
@@ -152,11 +167,12 @@ that holds it.
152
167
  ### Orphan process
153
168
 
154
169
  ```bash
155
- pgrep -fl 'autorag|autorag-gateway|minsync' | grep -v pgrep
170
+ pgrep -fl 'autorag|minsync' | grep -v pgrep
156
171
  ```
157
172
 
158
- A refresh that was killed mid-run can leave the gateway or a `minsync` child
159
- alive. Stop the gateway with `autorag gateway stop` first; only kill a PID
173
+ A refresh that was killed mid-run can leave a `minsync serve` daemon or a
174
+ `minsync` child alive. A daemon idle-stops itself; stop a confirmed orphan by
175
+ the pid from `~/.cache/minsync/serve/<instance>/pid`, and only kill a PID
160
176
  directly when it is confirmed orphaned (no parent CLI, no live refresh).
161
177
  Re-run `autorag status --json` afterwards to confirm `inFlight: false`.
162
178
 
@@ -167,16 +183,21 @@ were built with a different embedder than the configured one. Vectors of two
167
183
  different dimensions can never be compared, so the index must be rebuilt:
168
184
 
169
185
  ```bash
170
- autorag models prefetch --profile qwen3-embedding-0.6b
171
186
  autorag index rebuild --method minsync
172
187
  ```
173
188
 
174
- The current default is the local `native:Qwen/Qwen3-Embedding-0.6B` runtime at
175
- 1024 dimensions. A workspace still pinned to the legacy 768-dimension
176
- Ollama/TEI path must be reindexed explicitly, or pinned to an explicit profile.
177
- Changing `embedder.dimension` in config **without** a rebuild leaves retrieval
178
- silently empty. Keep embeddings local; do not point the embedder at a remote
179
- endpoint to work around a local failure.
189
+ The default is MinSync's local `Qwen/Qwen3-Embedding-0.6B` engine at 1024
190
+ dimensions, reached through a running `minsync serve` daemon or, failing that,
191
+ MinSync's in-process engine. Both report one identity (`provider: minsync`), so
192
+ a store never re-embeds because the daemon appeared or went away. Vectors
193
+ written by the removed local Q8_0 gateway engine are **not** compatible: their
194
+ identity changed, so the memory store (`memory.json.vectors.db`), chat-sidecar
195
+ semantic DBs, and the gist `vectors.json` re-embed on the next refresh, while a
196
+ MinSync workspace created through the old gateway profile needs
197
+ `autorag index rebuild --method minsync` (or
198
+ `autorag index reset --method minsync --yes` then refresh). Changing the
199
+ embedder's `dimension` in config **without** a rebuild leaves retrieval silently
200
+ empty.
180
201
 
181
202
  ### Stale index
182
203
 
@@ -208,7 +229,7 @@ to retry those files once without re-indexing healthy ones.
208
229
  (`cargo install minsync`) or leave `minSync.autoInstall` on, then retry.
209
230
  - `jikji-unavailable`: the binary is missing and auto-install failed. It
210
231
  installs through cargo; verify the Rust toolchain, then `autorag refresh --method jikji` to retry.
211
- - `auth-error` / `rate-limited`: model or datasource credentials. Report the
232
+ - `datasource-auth-error` / `datasource-rate-limited`: datasource credentials or quota. Report the
212
233
  missing environment-variable **name** and let the user supply it.
213
234
  - A configured datasource that returns nothing is a native store, connector, or
214
235
  query problem — every configured connection is searchable. Run its native
@@ -227,13 +248,12 @@ to retry those files once without re-indexing healthy ones.
227
248
  | `jikji-unavailable` | Jikji binary missing or install failed | Check cargo, retry refresh |
228
249
  | `everything-index-failed` | Windows Everything instance could not start or index; message carries ES exit code and stderr | Fix the reported cause, `autorag refresh --method everything --json` |
229
250
  | `embedding-identity-mismatch` | Indexed vectors use a different embedder | `autorag index rebuild --method minsync` |
230
- | `embedder-unavailable` | Embedding gateway or runtime down | `autorag gateway status --format json` |
231
- | `lock-conflict` | Another runtime holds the lock | `autorag gateway stop` |
251
+ | `embedder-unavailable` | No remote endpoint answered, no daemon could start, and the in-process addon is missing or failed | Run `minsync serve` by hand to read its error, install MinSync (`cargo install minsync`) or `@nomadamas/minsync-native`, or configure a remote `minSync.embedder` |
232
252
  | `datasource-index-failed` | Backing CLI failed to index | Run that CLI's own doctor |
233
253
  | `datasource-empty` | Store has no matching content | Re-sync with the CLI, or accept as empty |
234
254
  | `unknown-datasource-skill` | Config name is not a known template | Fix the name or add `"type"` |
235
255
  | `retrieval-method-failed` | One method errored during the query | Read `--debug` diagnostics |
236
- | `auth-error` | Credentials missing or rejected | Report the env var name |
256
+ | `datasource-auth-error` | Datasource credentials missing or rejected | Report the env var name |
237
257
  | `query-route-fallback` | Jev routing unavailable (often `OPENROUTER_API_KEY` unset); searched local with the original question | `test -n "$OPENROUTER_API_KEY"`; report the env var name, never its value |
238
258
  | `query-decomposition-failed` | Decomposition model call failed; searched the original question | Check `queryDecomposition.model` resolves (`autorag models list --provider openrouter`) |
239
259
  | `follow-up-check-fallback` | Jev post-fast-answer check unavailable; the run verified | Same as `query-route-fallback` |
@@ -168,14 +168,14 @@ AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite refresh --jso
168
168
  MinSync and Jikji auto-install during refresh (never during a query). If they
169
169
  are missing or broken, run a full refresh or return to setup rather than
170
170
  silently degrading to lexical-only search.
171
- - MinSync's default embedder is in-process native Qwen3 embeddings
172
- (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.10): no
173
- embedder flags, no API key, and no external daemon (such as Ollama) are
174
- needed, and no corpus text leaves the machine. For gateway-profiled
175
- embeddings, prefetch with `autorag models prefetch --profile qwen3-embedding-0.6b`.
176
- The legacy Ollama/TEI adapter path (EmbeddingGemma, 768 dimensions) is for
177
- manual QA only. Override the embedder only when intentionally using a
178
- different provider.
171
+ - AutoRAG owns no embedding model; MinSync provisions
172
+ `Qwen/Qwen3-Embedding-0.6B` (1024 dimensions) in `~/.cache/minsync/models`. A
173
+ configured remote `minSync.embedder` is used first; otherwise a running
174
+ `minsync serve` daemon (started by `refresh`/`watch`, never by a search), and
175
+ failing that MinSync's in-process engine via the optional
176
+ `@nomadamas/minsync-native` addon. The local default needs no embedder flags
177
+ and no API key. Override the embedder only when intentionally using a remote
178
+ provider.
179
179
  - Exact duplicate exclusion during refresh is enabled by default via the
180
180
  external `dupey` CLI. Install dupey during setup when it is missing
181
181
  (`command -v dupey || cargo install dupey --locked`) and tell the user the feature is
@@ -189,20 +189,28 @@ is impossible, refresh continues without this optimization and the user is
189
189
  told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
190
190
  index every copy.
191
191
 
192
- MinSync's default embedder is in-process native Qwen3 embeddings
193
- (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.10). The
194
- default needs no embedder flags, no API key, and no external daemon (such as
195
- Ollama) — all embeddings run in-process locally and privately. For workspaces
196
- using the loopback llama-server gateway, prefetch the verified model with
197
- `autorag models prefetch --profile qwen3-embedding-0.6b` (or verify with
198
- `autorag models verify --profile qwen3-embedding-0.6b`).
199
-
200
- The legacy Ollama/TEI adapter path (EmbeddingGemma, 768 dimensions) is
201
- supported only for existing legacy workspaces and manual QA; do not use it as a
202
- fresh install default.
203
-
204
- Override the embedder only when intentionally using a different, for example
205
- remote, provider:
192
+ AutoRAG owns no embedding model. MinSync provisions the embedder
193
+ (`Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions) in its own cache
194
+ (`~/.cache/minsync/models`); there is no local model download and no bundled
195
+ runtime. One shared embedder serves everything that embeds inside
196
+ AutoRAG: MinSync indexing/query, memory similar-question search, the
197
+ chat-crawler sidecars, the GitHub Gist vector index, and the endpoint external
198
+ CLIs are pointed at. The default needs no embedder flags and no API key. It
199
+ resolves in order:
200
+
201
+ 1. A configured remote endpoint (`minSync.embedder`), used for everything; no
202
+ daemon is started and no local model is loaded.
203
+ 2. A running `minsync serve` daemon (loopback only; MinSync 0.4.11+).
204
+ AutoRAG never starts it: run `minsync serve --idle-timeout 0` yourself, as
205
+ a launchd/systemd service (see `docs/minsync-setup.md`), or press
206
+ Ctrl+Shift+E in `autorag tui`. `autorag setup` only reports whether it is
207
+ alive.
208
+ 3. MinSync's in-process engine, through the optional Node-API addon
209
+ `@nomadamas/minsync-native` (same engine and weights, so bit-for-bit
210
+ identical vectors; not bundled). Without a daemon and without the addon,
211
+ embedding fails with an error naming the missing package.
212
+
213
+ Override the embedder only when intentionally using a remote provider:
206
214
 
207
215
  ```bash
208
216
  autorag init \
@@ -213,9 +221,14 @@ autorag init \
213
221
  --embedder-batch-size 64
214
222
  ```
215
223
 
216
- Only store the environment-variable name, never its value. Dimension and batch
217
- size must be positive integers, and the dimension must match the embedder
218
- (default Qwen3 is 1024; legacy EmbeddingGemma is 768; `voyageai/voyage-4-lite` via OpenRouter is 1024).
224
+ A remote `minSync.embedder` names an `id` (`openai:<model>`, `tei:<model>`, or
225
+ any id together with a `baseUrl`) plus optional `apiKeyEnv` (the **name** of the
226
+ environment variable holding the bearer token), `dimension`, `batchSize`, and
227
+ `timeoutMs`. OpenAI-protocol endpoints POST `{baseUrl}/v1/embeddings`; a `tei:`
228
+ endpoint POSTs `{baseUrl}/embed`; `openai:` with no `baseUrl` means
229
+ `https://api.openai.com`. Only store the environment-variable name, never its
230
+ value. `dimension` and `batchSize` must be positive integers, and `dimension`
231
+ must match the endpoint (the local default is 1024).
219
232
 
220
233
  ### Jev routing and question decomposition (on by default)
221
234
 
@@ -336,7 +349,7 @@ keeps its mirrors and reports `search-root-unavailable`; see
336
349
 
337
350
  ## Probe and configure datasource skills (setup wizard)
338
351
 
339
- Use `autorag setup` to probe the local runtime, model profile, and known
352
+ Use `autorag setup` to probe the embedding backend, MinSync, and known
340
353
  datasources automatically:
341
354
 
342
355
  ```bash
@@ -393,13 +406,15 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
393
406
  Tokens are environment-variable names, not raw secrets. CLI-backed connectors
394
407
  keep authentication in their external tool configuration.
395
408
 
396
- Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.0` or newer)
409
+ Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.2` or newer)
397
410
  and configured through its own Himalaya account. AutoRAG runs its local `sync`
398
411
  and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
399
412
  hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
400
- fails with `text array must be non-empty`. 0.2.0 defaults to the in-process
401
- native `Qwen/Qwen3-Embedding-0.6B` embedder and keeps vectors in LanceDB, so a
402
- cold cache makes the first `index` download ONNX weights and run for minutes.
413
+ fails with `text array must be non-empty`. 0.2.2 embeds through the same
414
+ backend as the rest of AutoRAG (a remote `minSync.embedder`, else the
415
+ `minsync serve` daemon) instead of its own in-process model, and keeps vectors
416
+ in LanceDB. Switching from the old native model rebuilds the semantic index on
417
+ the next `index`. kakao needs `lazykatok` 0.3.3 or newer for the same reason.
403
418
  Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
404
419
  datasource is removed.
405
420
 
@@ -415,7 +430,8 @@ autorag refresh --json
415
430
  autorag search "summarize the collection" --top-k 3 --json --debug
416
431
  ```
417
432
 
418
- - `setup` probes runtime health, model profile, and datasource readiness.
433
+ - `setup` probes the embedding backend (`embedding.backend`), MinSync, and
434
+ datasource readiness.
419
435
  - `status` is model-free and path-opaque.
420
436
  - `health` resolves the single model, checks credential presence, and normally
421
437
  performs one live completion probe.
@@ -1,19 +0,0 @@
1
- import type { AgentTool } from "@earendil-works/pi-agent-core";
2
- import { Type } from "typebox";
3
- export declare const RECOMMEND_PEER_TARGETS_TOOL_NAME = "recommend_peer_targets";
4
- declare const recommendPeerTargetsSchema: Type.TObject<{
5
- query: Type.TString;
6
- }>;
7
- export interface PeerTargetRecommendation {
8
- readonly alias: string;
9
- readonly matchedTerms: readonly string[];
10
- readonly displayName?: string;
11
- readonly description?: string;
12
- }
13
- export interface RecommendPeerTargetsDetails {
14
- readonly method: "recommend_peer_targets";
15
- readonly resultCount: number;
16
- readonly matches: readonly PeerTargetRecommendation[];
17
- }
18
- export declare function createRecommendPeerTargetsTool(workspacePath: string): AgentTool<typeof recommendPeerTargetsSchema, RecommendPeerTargetsDetails>;
19
- export {};
@@ -1,51 +0,0 @@
1
- import type { AgentTool } from "@earendil-works/pi-agent-core";
2
- import { Type } from "typebox";
3
- import { type SimplexQueryState } from "../p2p/simplex-query-store.js";
4
- import type { SimplexTransport } from "../p2p/simplex-transport.js";
5
- import type { PeerQueryResponse } from "../p2p/wire.js";
6
- import { EvidenceLedger } from "./evidence-ledger.js";
7
- export declare const QUERY_PEER_AGENT_TOOL_NAME = "query_peer_agent";
8
- declare const queryPeerAgentSchema: Type.TObject<{
9
- alias: Type.TString;
10
- query: Type.TString;
11
- topK: Type.TOptional<Type.TInteger>;
12
- scope: Type.TOptional<Type.TString>;
13
- }>;
14
- export interface QueryPeerAgentResult {
15
- readonly number: number;
16
- readonly title: string;
17
- readonly summary: string;
18
- readonly source: string;
19
- readonly excerpt: string;
20
- }
21
- export interface QueryPeerAgentDetails {
22
- readonly method: "query_peer_agent";
23
- readonly ok: boolean;
24
- readonly alias: string;
25
- readonly contactId?: number;
26
- readonly pendingId?: string;
27
- readonly expiresAt?: string;
28
- readonly status?: "ok" | "rejected" | "pending";
29
- readonly answer?: string;
30
- readonly results?: readonly QueryPeerAgentResult[];
31
- readonly diagnostics?: PeerQueryResponse["diagnostics"];
32
- readonly fileCount?: number;
33
- readonly message?: string;
34
- }
35
- export interface QueryPeerAgentToolOptions {
36
- readonly workspacePath: string;
37
- readonly openTransport: () => Promise<SimplexTransport>;
38
- readonly fastTimeoutMs?: number;
39
- readonly autoStart?: boolean;
40
- readonly sessionId?: () => string | undefined;
41
- readonly onResponse?: (state: SimplexQueryState, response: PeerQueryResponse) => void;
42
- readonly onExpired?: (state: SimplexQueryState) => void;
43
- /** Run evidence ledger; each peer result is registered so the final emit can cite it by id. */
44
- readonly ledger?: EvidenceLedger;
45
- }
46
- export interface QueryPeerAgentTool extends AgentTool<typeof queryPeerAgentSchema, QueryPeerAgentDetails> {
47
- readonly ready: Promise<void>;
48
- close(): Promise<void>;
49
- }
50
- export declare function createQueryPeerAgentTool(options: QueryPeerAgentToolOptions): QueryPeerAgentTool;
51
- export {};
@@ -1,7 +0,0 @@
1
- import { type RuntimeStatus } from "../../embedding-runtime/index.js";
2
- import type { CommandContext } from "./types.js";
3
- export interface GatewayCommandDeps {
4
- runtimeStatus?: () => Promise<RuntimeStatus>;
5
- stopRuntime?: () => Promise<void>;
6
- }
7
- export declare function runGateway(ctx: CommandContext, deps?: GatewayCommandDeps): Promise<number>;
@@ -1,2 +0,0 @@
1
- import type { CommandContext } from "./types.js";
2
- export declare function runP2pPolicy(ctx: CommandContext): number;
@@ -1,7 +0,0 @@
1
- import type { CommandContext } from "./types.js";
2
- /** Resolve the workspace used for P2P identity/policy, matching `autorag serve`. */
3
- export declare function resolveP2pWorkspace(ctx: CommandContext): string;
4
- /**
5
- * `autorag p2p` — SimpleX peer trust management.
6
- */
7
- export declare function runP2p(ctx: CommandContext): Promise<number>;