@autorag/librarian 2.10.5 → 2.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -9
- package/README.md +19 -18
- package/dist/agent/agent.d.ts +45 -48
- package/dist/agent/context-token-guard.d.ts +11 -0
- package/dist/agent/discord-remote.d.ts +30 -0
- package/dist/agent/embedding-controls.d.ts +22 -0
- package/dist/agent/index.d.ts +1 -2
- package/dist/agent/pi-session.d.ts +0 -1
- package/dist/agent/query-decomposition.d.ts +1 -1
- package/dist/agent/search-documents.d.ts +2 -2
- package/dist/agent/system-prompt.d.ts +1 -1
- package/dist/cli/commands/models.d.ts +0 -14
- package/dist/cli/commands/search.d.ts +3 -4
- package/dist/cli/config.d.ts +7 -19
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +3134 -6671
- package/dist/cli/setup.d.ts +13 -36
- package/dist/cloud/config.d.ts +3 -3
- package/dist/core.js +5144 -7954
- package/dist/datasource/embedding-env.d.ts +29 -0
- package/dist/datasource/sidecar/config.d.ts +3 -0
- package/dist/datasource/sidecar/index.d.ts +1 -1
- package/dist/datasource/skills/discrawl/client.d.ts +1 -0
- package/dist/datasource/skills/discrawl/embedding.d.ts +17 -0
- package/dist/datasource/skills/discrawl/types.d.ts +11 -3
- package/dist/datasource/skills/factory.d.ts +14 -1
- package/dist/datasource/skills/github-gist/semantic.d.ts +5 -5
- package/dist/datasource/skills/github-gist/skill.d.ts +2 -3
- package/dist/datasource/skills/lazykatok/types.d.ts +9 -1
- package/dist/datasource/skills/mailcrawl/types.d.ts +9 -1
- package/dist/embedding/daemon.d.ts +57 -0
- package/dist/embedding/embedder.d.ts +59 -0
- package/dist/embedding/http-embedder.d.ts +28 -0
- package/dist/embedding/in-process.d.ts +19 -0
- package/dist/embedding/index.d.ts +28 -0
- package/dist/embedding/managed.d.ts +44 -0
- package/dist/embedding/prefixes.d.ts +18 -0
- package/dist/embedding/settings.d.ts +9 -0
- package/dist/index.js +5153 -7965
- package/dist/mcp/index.js +4890 -7700
- package/dist/memory/context.d.ts +1 -1
- package/dist/memory/similar-queries.d.ts +1 -1
- package/dist/minsync/binary.d.ts +21 -0
- package/dist/minsync/client.d.ts +11 -10
- package/dist/minsync/index.d.ts +1 -1
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/method.d.ts +3 -11
- package/dist/minsync/process.d.ts +1 -0
- package/dist/minsync/types.d.ts +0 -2
- package/dist/retrieval/rerank.d.ts +10 -3
- package/package.json +3 -3
- package/skills/autorag/SKILL.md +3 -8
- package/skills/autorag-doctor/SKILL.md +46 -26
- package/skills/autorag-lite-setup/SKILL.md +8 -8
- package/skills/autorag-setup/SKILL.md +39 -23
- package/dist/agent/peer-target-tool.d.ts +0 -19
- package/dist/agent/query-peer-tool.d.ts +0 -51
- package/dist/cli/commands/gateway.d.ts +0 -7
- package/dist/cli/commands/p2p-policy.d.ts +0 -2
- package/dist/cli/commands/p2p.d.ts +0 -7
- package/dist/cli/commands/serve.d.ts +0 -19
- package/dist/embedding-runtime/cache.d.ts +0 -30
- package/dist/embedding-runtime/gateway-embedder.d.ts +0 -41
- package/dist/embedding-runtime/gateway.d.ts +0 -27
- package/dist/embedding-runtime/index.d.ts +0 -120
- package/dist/embedding-runtime/manifest.d.ts +0 -20
- package/dist/embedding-runtime/supervisor.d.ts +0 -74
- package/dist/embedding-runtime/types.d.ts +0 -70
- package/dist/p2p/approval-store.d.ts +0 -32
- package/dist/p2p/egress-gate.d.ts +0 -22
- package/dist/p2p/file-sharing.d.ts +0 -33
- package/dist/p2p/injection-classifier.d.ts +0 -33
- package/dist/p2p/injection-gate.d.ts +0 -37
- package/dist/p2p/pii-gate.d.ts +0 -47
- package/dist/p2p/policy-filter.d.ts +0 -15
- package/dist/p2p/policy.d.ts +0 -77
- package/dist/p2p/simplex-query-store.d.ts +0 -44
- package/dist/p2p/simplex-server.d.ts +0 -115
- package/dist/p2p/simplex-transport.d.ts +0 -62
- package/dist/p2p/wire.d.ts +0 -39
- package/licenses/embedding-assets.json +0 -85
- package/licenses/llama.cpp-MIT.txt +0 -21
- package/licenses/qwen3-embedding-notice.txt +0 -13
package/dist/memory/context.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Embedder } from "../embedding
|
|
1
|
+
import type { Embedder } from "../embedding/embedder.js";
|
|
2
2
|
import type { JudgedEvidenceRecord } from "./judged-evidence.js";
|
|
3
3
|
import type { RetrievalInsight, RetrievalMemory } from "./memory.js";
|
|
4
4
|
import { type SimilarQuestion } from "./similar-queries.js";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Embedder } from "../embedding
|
|
1
|
+
import type { Embedder } from "../embedding/embedder.js";
|
|
2
2
|
import type { JudgedEvidenceRecord } from "./judged-evidence.js";
|
|
3
3
|
/** How many similar past questions memory brings back for one question. */
|
|
4
4
|
export declare const SIMILAR_QUESTION_LIMIT = 25;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type EnsureMinSyncBinaryOptions } from "./installer.js";
|
|
2
|
+
export interface ResolveMinSyncBinaryOptions {
|
|
3
|
+
/** Workspace project root; the cached binary lives under `<root>/.autorag/bin`. */
|
|
4
|
+
readonly root: string;
|
|
5
|
+
/** Test/embedded override; CLI config intentionally does not persist this. */
|
|
6
|
+
readonly binaryPath?: string;
|
|
7
|
+
readonly autoInstall?: boolean;
|
|
8
|
+
readonly installer?: Omit<EnsureMinSyncBinaryOptions, "root">;
|
|
9
|
+
/** False for query paths: they use an existing binary or fail, and never wait on `cargo install`. */
|
|
10
|
+
readonly install?: boolean;
|
|
11
|
+
}
|
|
12
|
+
/** Resolve the `minsync` executable from PATH directories. Returns the first match or undefined. */
|
|
13
|
+
export declare function lookupMinSyncInPath(env: NodeJS.ProcessEnv): string | undefined;
|
|
14
|
+
/**
|
|
15
|
+
* Resolve the MinSync CLI from the user's global PATH first, then the workspace cache, and finally the
|
|
16
|
+
* verified auto-install fallback. Throws {@link MinSyncRequiredError}: MinSync is required, so there is
|
|
17
|
+
* no "no binary" value to degrade on.
|
|
18
|
+
*/
|
|
19
|
+
export declare function resolveMinSyncBinary(options: ResolveMinSyncBinaryOptions): Promise<string>;
|
|
20
|
+
/** True when no `minsync` is resolvable right now without installing one. */
|
|
21
|
+
export declare function isMinSyncBinaryMissing(options: Pick<ResolveMinSyncBinaryOptions, "root" | "binaryPath">): boolean;
|
package/dist/minsync/client.d.ts
CHANGED
|
@@ -1,18 +1,9 @@
|
|
|
1
|
-
import type { EnsuredRuntime } from "../embedding-runtime/index.js";
|
|
2
1
|
import type { MinSyncEmbedderConfig, MinSyncQueryHit, MinSyncSyncResult } from "./types.js";
|
|
3
|
-
export declare const MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.";
|
|
4
|
-
export interface MinSyncRuntime {
|
|
5
|
-
ensureRuntime(options?: {
|
|
6
|
-
readonly profileId?: MinSyncEmbedderConfig["profile"];
|
|
7
|
-
readonly cachedOnly?: boolean;
|
|
8
|
-
}): Promise<EnsuredRuntime>;
|
|
9
|
-
}
|
|
10
2
|
export interface MinSyncClientOptions {
|
|
11
3
|
readonly binaryPath: string;
|
|
12
4
|
readonly workspacePath: string;
|
|
13
5
|
readonly embedder?: MinSyncEmbedderConfig;
|
|
14
6
|
readonly maxChunkSize?: number;
|
|
15
|
-
readonly runtime?: MinSyncRuntime;
|
|
16
7
|
}
|
|
17
8
|
/** MinSync's native query modes: vector, BM25, and hybrid. */
|
|
18
9
|
export type MinSyncQueryMode = "vector" | "bm25" | "hybrid";
|
|
@@ -31,8 +22,12 @@ export declare class MinSyncClient {
|
|
|
31
22
|
private readonly workspacePath;
|
|
32
23
|
private readonly embedder;
|
|
33
24
|
private readonly maxChunkSize;
|
|
34
|
-
private readonly runtime;
|
|
35
25
|
constructor(options: MinSyncClientOptions);
|
|
26
|
+
/**
|
|
27
|
+
* What MinSync is told to embed with. No `id`, or a `native:` id, is MinSync's own Qwen3 engine, which
|
|
28
|
+
* MinSync provisions and runs (in-process, or through a `minsync serve` daemon it finds on its own).
|
|
29
|
+
* Anything else is an endpoint the operator configured, passed through untouched.
|
|
30
|
+
*/
|
|
36
31
|
private effectiveEmbedder;
|
|
37
32
|
private readIdentity;
|
|
38
33
|
private writeIdentity;
|
|
@@ -50,4 +45,10 @@ export declare class MinSyncClient {
|
|
|
50
45
|
private checkAndSync;
|
|
51
46
|
query(text: string, topK: number, mode?: MinSyncQueryMode, signal?: AbortSignal): Promise<readonly MinSyncQueryHit[]>;
|
|
52
47
|
private spawn;
|
|
48
|
+
/**
|
|
49
|
+
* MinSync's `openai:` embedder reads its bearer token from `OPENAI_API_KEY` only. The operator names the
|
|
50
|
+
* variable that holds their key with `apiKeyEnv`, so that value is what the child receives as
|
|
51
|
+
* `OPENAI_API_KEY`. A local `native:` embedder needs no key and the environment is left as it is.
|
|
52
|
+
*/
|
|
53
|
+
private childEnv;
|
|
53
54
|
}
|
package/dist/minsync/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { MinSyncClient, type MinSyncClientOptions, MinSyncQueryError, type MinSyncQueryMode, } from "./client.js";
|
|
2
2
|
export { configuredVectorDimension, DEFAULT_MINSYNC_EMBEDDER_CONFIG, DEFAULT_MINSYNC_EMBEDDER_DIMENSION, DEFAULT_MINSYNC_EMBEDDER_ID, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE, type MinSyncEmbeddingIdentity, minSyncConfigPath, minSyncEmbeddingIdentityPath, rewriteEmbedderConfig, } from "./embedder-config.js";
|
|
3
3
|
export { MinSyncRequiredError } from "./errors.js";
|
|
4
4
|
export { type EnsureMinSyncBinaryOptions, ensureMinSyncBinary, executableName, fetchLatestMinSyncRelease, type InstalledMinSyncBinary, MINSYNC_VERSION, type MinSyncRelease, type MinSyncReleaseAsset, MinSyncReleaseError, selectReleaseAsset, } from "./installer.js";
|
package/dist/minsync/method.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
|
|
2
2
|
import type { MinSyncQueryMode } from "./client.js";
|
|
3
|
-
import {
|
|
4
|
-
import { type EnsureMinSyncBinaryOptions } from "./installer.js";
|
|
3
|
+
import type { EnsureMinSyncBinaryOptions } from "./installer.js";
|
|
5
4
|
import type { MinSyncEmbedderConfig, MinSyncSyncResult } from "./types.js";
|
|
6
5
|
export interface MinSyncVectorMethodOptions {
|
|
7
6
|
readonly root: string;
|
|
@@ -13,7 +12,6 @@ export interface MinSyncVectorMethodOptions {
|
|
|
13
12
|
readonly embedder?: MinSyncEmbedderConfig;
|
|
14
13
|
readonly maxChunkSize?: number;
|
|
15
14
|
readonly mode?: MinSyncQueryMode;
|
|
16
|
-
readonly runtime?: MinSyncRuntime;
|
|
17
15
|
/** Default `topK` used when the caller omits one. Default 50. */
|
|
18
16
|
readonly defaultTopK?: number;
|
|
19
17
|
/** Fetch cap applied when a scope narrows the query. Default 100. */
|
|
@@ -29,7 +27,6 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
|
|
|
29
27
|
private readonly embedder;
|
|
30
28
|
private readonly maxChunkSize;
|
|
31
29
|
private readonly mode;
|
|
32
|
-
private readonly runtime;
|
|
33
30
|
private readonly defaultTopK;
|
|
34
31
|
private readonly scopedTopK;
|
|
35
32
|
constructor(options: MinSyncVectorMethodOptions);
|
|
@@ -45,13 +42,8 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
|
|
|
45
42
|
retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
|
|
46
43
|
private retrieveUnlocked;
|
|
47
44
|
/**
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* auto-install runs only for `sync` (refresh); queries pass
|
|
51
|
-
* `install: false` so a user's turn never waits on `cargo install`.
|
|
52
|
-
*
|
|
53
|
-
* Returns the executable path, or throws {@link MinSyncRequiredError}:
|
|
54
|
-
* MinSync is required, so there is no "no binary" value to degrade on.
|
|
45
|
+
* Queries pass `install: false` so a user's turn never waits on `cargo install`; only `sync` (refresh)
|
|
46
|
+
* may install. See {@link resolveMinSyncBinary}.
|
|
55
47
|
*/
|
|
56
48
|
protected resolveBinary(options?: {
|
|
57
49
|
readonly install?: boolean;
|
package/dist/minsync/types.d.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
/** Embedder configuration for MinSync vector indexing. Shared by config and method layers. */
|
|
2
2
|
import type { MinSyncLexicalLanguage } from "./lexical-language.js";
|
|
3
3
|
export interface MinSyncEmbedderConfig {
|
|
4
|
-
/** Select an AutoRAG-owned embedding runtime profile. */
|
|
5
|
-
readonly profile?: "qwen3-embedding-0.6b";
|
|
6
4
|
readonly id?: string;
|
|
7
5
|
readonly baseUrl?: string;
|
|
8
6
|
/** Environment variable name whose value holds the embedder API key. /^[A-Za-z_][A-Za-z0-9_]*$/ */
|
|
@@ -24,6 +24,13 @@ export declare const DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY";
|
|
|
24
24
|
* chat LLMs, so the project's 2026+ model rule does not cover them.
|
|
25
25
|
*/
|
|
26
26
|
export declare const DEFAULT_DAZZI_RERANK_MODEL = "qwen/qwen3-reranker-8b";
|
|
27
|
+
/**
|
|
28
|
+
* Default per-request rerank timeout. A reranker that never answers would stall
|
|
29
|
+
* the fast answer. Measured against the hosted reranker, a 172-document pool
|
|
30
|
+
* takes ~15s and a 300-document pool ~45s, so 60s clears the largest baseline
|
|
31
|
+
* pool (3 x 100 candidates) with headroom.
|
|
32
|
+
*/
|
|
33
|
+
export declare const DEFAULT_RERANK_TIMEOUT_MS = 60000;
|
|
27
34
|
/** Default number of merged chunks kept after reranking. */
|
|
28
35
|
export declare const DEFAULT_RERANK_TOP_N = 25;
|
|
29
36
|
/** Rerank providers {@link createReranker} can build. */
|
|
@@ -65,7 +72,7 @@ export interface Reranker {
|
|
|
65
72
|
*/
|
|
66
73
|
rerank(query: string, results: readonly RetrievalResult[], options?: RerankOptions): Promise<RetrievalResult[]>;
|
|
67
74
|
}
|
|
68
|
-
/** Minimal rerank client shape, so tests can inject a fake without
|
|
75
|
+
/** Minimal rerank client shape, so tests can inject a fake without network access. */
|
|
69
76
|
export interface RerankClient {
|
|
70
77
|
rerank(request: {
|
|
71
78
|
requestBody: {
|
|
@@ -78,7 +85,7 @@ export interface RerankClient {
|
|
|
78
85
|
signal?: AbortSignal;
|
|
79
86
|
}): Promise<RerankResponse>;
|
|
80
87
|
}
|
|
81
|
-
/**
|
|
88
|
+
/** Rerank response as {@link OpenRouterReranker} consumes it. */
|
|
82
89
|
export interface RerankResponse {
|
|
83
90
|
readonly results: ReadonlyArray<{
|
|
84
91
|
readonly index: number;
|
|
@@ -123,7 +130,7 @@ export interface CreateRerankerOptions extends OpenRouterRerankerOptions {
|
|
|
123
130
|
*/
|
|
124
131
|
export declare function createReranker(config: CreateRerankerOptions | false | undefined): Reranker | undefined;
|
|
125
132
|
/**
|
|
126
|
-
* Reranker backed by an OpenRouter-compatible rerank
|
|
133
|
+
* Reranker backed by an OpenRouter-compatible rerank endpoint.
|
|
127
134
|
*
|
|
128
135
|
* OpenRouter is the default provider; the hosted Dazzi Models plan serves the
|
|
129
136
|
* same wire shape at `<root>/v1/rerank`, so {@link createReranker} reuses this
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@autorag/librarian",
|
|
3
|
-
"version": "2.10.
|
|
3
|
+
"version": "2.10.6",
|
|
4
4
|
"description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -44,7 +44,6 @@
|
|
|
44
44
|
"check": "biome check --write .",
|
|
45
45
|
"lint": "biome check .",
|
|
46
46
|
"typecheck": "tsc --noEmit",
|
|
47
|
-
"check:embedding-manifest": "node scripts/check-embedding-manifest.mjs",
|
|
48
47
|
"supply-chain": "bun scripts/supply-chain/evaluate.ts gate --sbom sbom.local.cdx.json"
|
|
49
48
|
},
|
|
50
49
|
"engines": {
|
|
@@ -57,8 +56,8 @@
|
|
|
57
56
|
"@earendil-works/pi-tui": "1.2.0",
|
|
58
57
|
"@llblab/pi-telegram": "0.57.2",
|
|
59
58
|
"@modelcontextprotocol/server": "^2.3.1",
|
|
60
|
-
"@openrouter/sdk": "^1.4.25",
|
|
61
59
|
"chardet": "^2.2.0",
|
|
60
|
+
"discord.js": "14.27.0",
|
|
62
61
|
"fast-xml-parser": "^5.11.2",
|
|
63
62
|
"iconv-lite": "^0.7.3",
|
|
64
63
|
"jev-use": "^0.8.0",
|
|
@@ -66,6 +65,7 @@
|
|
|
66
65
|
"kordoc": "^4.21.30",
|
|
67
66
|
"mailparser": "^3.9.38",
|
|
68
67
|
"pdfjs-dist": "^6.4.299",
|
|
68
|
+
"pi-discord-remote": "0.2.4",
|
|
69
69
|
"puppeteer-core": "^25.13.0",
|
|
70
70
|
"slides-grab": "^1.5.2",
|
|
71
71
|
"smol-toml": "1.9.1",
|
package/skills/autorag/SKILL.md
CHANGED
|
@@ -142,8 +142,6 @@ autorag index rebuild --yes --json
|
|
|
142
142
|
autorag index reset --method parsed --yes --json
|
|
143
143
|
autorag memory inspect --json
|
|
144
144
|
autorag tui
|
|
145
|
-
autorag serve --force
|
|
146
|
-
autorag p2p policy list --json
|
|
147
145
|
```
|
|
148
146
|
|
|
149
147
|
Prefer a full refresh so parsed mirrors, MinSync, Jikji, configured
|
|
@@ -153,9 +151,7 @@ aligned. `--method` accepts
|
|
|
153
151
|
a `--method` name. Use `--method` only for deliberate narrowing. Scheduled
|
|
154
152
|
maintenance should use non-daemon `autorag watch --once`, typically every
|
|
155
153
|
1 hour, with the same config used by search and no overlapping runs.
|
|
156
|
-
`autorag tui` is the shipped beta terminal UI.
|
|
157
|
-
are opt-in SimpleX peer sharing (disabled until `p2p.enabled` is true, unless
|
|
158
|
-
`--force`).
|
|
154
|
+
`autorag tui` is the shipped beta terminal UI.
|
|
159
155
|
|
|
160
156
|
Reset and rebuild commands remove only selected workspace `.autorag` indexes.
|
|
161
157
|
Never target source documents. `memory inspect` is read-only and path-opaque.
|
|
@@ -169,8 +165,7 @@ Never target source documents. `memory inspect` is read-only and path-opaque.
|
|
|
169
165
|
- Preserve real source mapping.
|
|
170
166
|
- Prefer `--json --debug` when another agent consumes search output.
|
|
171
167
|
- Do not invent CLI commands. `autorag --help` is the command list of record,
|
|
172
|
-
including the shipped `setup`, `
|
|
168
|
+
including the shipped `setup`, `models`, `tui`, and `lite`
|
|
173
169
|
commands. `autorag <command> --help` prints that command's own flags, and
|
|
174
170
|
`autorag --version` prints the installed version. See
|
|
175
|
-
`docs/
|
|
176
|
-
and gateway setup.
|
|
171
|
+
`docs/minsync-setup.md#embedding-backends` for MinSync setup and embedding-backend selection.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: autorag-doctor
|
|
3
|
-
description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding
|
|
3
|
+
description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
|
|
4
4
|
license: Apache-2.0
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -29,7 +29,7 @@ Always finish with the status table in [Report](#report).
|
|
|
29
29
|
```bash
|
|
30
30
|
autorag status --json
|
|
31
31
|
autorag health --json
|
|
32
|
-
autorag
|
|
32
|
+
autorag setup --format json
|
|
33
33
|
```
|
|
34
34
|
|
|
35
35
|
- `status` reports `state`, `stale`, `diagnostics`, and per-component state for
|
|
@@ -38,8 +38,12 @@ autorag gateway status --format json
|
|
|
38
38
|
- `health` resolves the single search model and does one live completion probe.
|
|
39
39
|
`--skip-probes` only proves config shape, never live access; do not claim a
|
|
40
40
|
healthy model from it.
|
|
41
|
-
-
|
|
42
|
-
`
|
|
41
|
+
- `setup --format json` reports `embedding.backend` (`remote`, `daemon`, or
|
|
42
|
+
`in-process`) and the resolved `provider`/`model`/`dimension` when a backend
|
|
43
|
+
is alive, or `embedding.reason` and a `remediation` when none is. It only
|
|
44
|
+
probes; it never starts a daemon. Semantic search needs the daemon (or a
|
|
45
|
+
remote endpoint) to be running: a backend that is absent is a real
|
|
46
|
+
`embedding-unavailable` state, not normal.
|
|
43
47
|
|
|
44
48
|
Config lives at `--config`, `AUTORAG_CONFIG`, `$AUTORAG_HOME/config.json`, or
|
|
45
49
|
`~/.autorag/config.json`. Read `searchPaths`, `workspacePath`, `minSync`,
|
|
@@ -129,19 +133,30 @@ autorag search "summarize the collection" --top-k 3 --json --debug
|
|
|
129
133
|
|
|
130
134
|
## 4. Repair playbook
|
|
131
135
|
|
|
132
|
-
###
|
|
136
|
+
### Stuck embedding daemon
|
|
133
137
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
138
|
+
AutoRAG owns no lock or pid file for embeddings. A running `minsync serve`
|
|
139
|
+
publishes `host`, `port`, and `pid` under `~/.cache/minsync/serve/<instance>/`
|
|
140
|
+
and removes them on a clean stop. A leftover file from a crashed daemon is
|
|
141
|
+
harmless: AutoRAG only adopts a daemon whose `GET /health` answers as a loopback
|
|
142
|
+
server of the wanted model, so a stale entry is skipped.
|
|
137
143
|
|
|
138
144
|
```bash
|
|
139
|
-
|
|
140
|
-
autorag
|
|
145
|
+
ls ~/.cache/minsync/serve/* 2>/dev/null
|
|
146
|
+
autorag setup --format json
|
|
141
147
|
```
|
|
142
148
|
|
|
143
|
-
|
|
144
|
-
|
|
149
|
+
To diagnose an embedding backend that is not alive, `setup --format json` and
|
|
150
|
+
`status` say so (`embedding.reason`, and the `embedding-unavailable`
|
|
151
|
+
diagnostic). AutoRAG never starts the daemon: run `minsync serve
|
|
152
|
+
--idle-timeout 0` by hand (or as a launchd/systemd service, see
|
|
153
|
+
`docs/minsync-setup.md`) and read its own error, or press Ctrl+Shift+E in the
|
|
154
|
+
TUI. When no daemon answers and the in-process addon
|
|
155
|
+
(`@nomadamas/minsync-native`) is missing, that is the `embedder-unavailable`
|
|
156
|
+
state. The daemon stops itself after `--idle-timeout` seconds (default 300), so
|
|
157
|
+
a semantic failure after a quiet spell usually means it timed out. Stop an
|
|
158
|
+
orphaned daemon by the pid from `~/.cache/minsync/serve/<instance>/pid` after
|
|
159
|
+
confirming no live refresh holds it.
|
|
145
160
|
|
|
146
161
|
Index locks are owned by their engines: MinSync/tantivy locks under
|
|
147
162
|
`<workspace>/.autorag/`, and per-CLI locks such as
|
|
@@ -152,11 +167,12 @@ that holds it.
|
|
|
152
167
|
### Orphan process
|
|
153
168
|
|
|
154
169
|
```bash
|
|
155
|
-
pgrep -fl 'autorag|
|
|
170
|
+
pgrep -fl 'autorag|minsync' | grep -v pgrep
|
|
156
171
|
```
|
|
157
172
|
|
|
158
|
-
A refresh that was killed mid-run can leave
|
|
159
|
-
alive.
|
|
173
|
+
A refresh that was killed mid-run can leave a `minsync serve` daemon or a
|
|
174
|
+
`minsync` child alive. A daemon idle-stops itself; stop a confirmed orphan by
|
|
175
|
+
the pid from `~/.cache/minsync/serve/<instance>/pid`, and only kill a PID
|
|
160
176
|
directly when it is confirmed orphaned (no parent CLI, no live refresh).
|
|
161
177
|
Re-run `autorag status --json` afterwards to confirm `inFlight: false`.
|
|
162
178
|
|
|
@@ -167,16 +183,21 @@ were built with a different embedder than the configured one. Vectors of two
|
|
|
167
183
|
different dimensions can never be compared, so the index must be rebuilt:
|
|
168
184
|
|
|
169
185
|
```bash
|
|
170
|
-
autorag models prefetch --profile qwen3-embedding-0.6b
|
|
171
186
|
autorag index rebuild --method minsync
|
|
172
187
|
```
|
|
173
188
|
|
|
174
|
-
The
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
189
|
+
The default is MinSync's local `Qwen/Qwen3-Embedding-0.6B` engine at 1024
|
|
190
|
+
dimensions, reached through a running `minsync serve` daemon or, failing that,
|
|
191
|
+
MinSync's in-process engine. Both report one identity (`provider: minsync`), so
|
|
192
|
+
a store never re-embeds because the daemon appeared or went away. Vectors
|
|
193
|
+
written by the removed local Q8_0 gateway engine are **not** compatible: their
|
|
194
|
+
identity changed, so the memory store (`memory.json.vectors.db`), chat-sidecar
|
|
195
|
+
semantic DBs, and the gist `vectors.json` re-embed on the next refresh, while a
|
|
196
|
+
MinSync workspace created through the old gateway profile needs
|
|
197
|
+
`autorag index rebuild --method minsync` (or
|
|
198
|
+
`autorag index reset --method minsync --yes` then refresh). Changing the
|
|
199
|
+
embedder's `dimension` in config **without** a rebuild leaves retrieval silently
|
|
200
|
+
empty.
|
|
180
201
|
|
|
181
202
|
### Stale index
|
|
182
203
|
|
|
@@ -208,7 +229,7 @@ to retry those files once without re-indexing healthy ones.
|
|
|
208
229
|
(`cargo install minsync`) or leave `minSync.autoInstall` on, then retry.
|
|
209
230
|
- `jikji-unavailable`: the binary is missing and auto-install failed. It
|
|
210
231
|
installs through cargo; verify the Rust toolchain, then `autorag refresh --method jikji` to retry.
|
|
211
|
-
- `auth-error` / `rate-limited`:
|
|
232
|
+
- `datasource-auth-error` / `datasource-rate-limited`: datasource credentials or quota. Report the
|
|
212
233
|
missing environment-variable **name** and let the user supply it.
|
|
213
234
|
- A configured datasource that returns nothing is a native store, connector, or
|
|
214
235
|
query problem — every configured connection is searchable. Run its native
|
|
@@ -227,13 +248,12 @@ to retry those files once without re-indexing healthy ones.
|
|
|
227
248
|
| `jikji-unavailable` | Jikji binary missing or install failed | Check cargo, retry refresh |
|
|
228
249
|
| `everything-index-failed` | Windows Everything instance could not start or index; message carries ES exit code and stderr | Fix the reported cause, `autorag refresh --method everything --json` |
|
|
229
250
|
| `embedding-identity-mismatch` | Indexed vectors use a different embedder | `autorag index rebuild --method minsync` |
|
|
230
|
-
| `embedder-unavailable` |
|
|
231
|
-
| `lock-conflict` | Another runtime holds the lock | `autorag gateway stop` |
|
|
251
|
+
| `embedder-unavailable` | No remote endpoint answered, no daemon could start, and the in-process addon is missing or failed | Run `minsync serve` by hand to read its error, install MinSync (`cargo install minsync`) or `@nomadamas/minsync-native`, or configure a remote `minSync.embedder` |
|
|
232
252
|
| `datasource-index-failed` | Backing CLI failed to index | Run that CLI's own doctor |
|
|
233
253
|
| `datasource-empty` | Store has no matching content | Re-sync with the CLI, or accept as empty |
|
|
234
254
|
| `unknown-datasource-skill` | Config name is not a known template | Fix the name or add `"type"` |
|
|
235
255
|
| `retrieval-method-failed` | One method errored during the query | Read `--debug` diagnostics |
|
|
236
|
-
| `auth-error` |
|
|
256
|
+
| `datasource-auth-error` | Datasource credentials missing or rejected | Report the env var name |
|
|
237
257
|
| `query-route-fallback` | Jev routing unavailable (often `OPENROUTER_API_KEY` unset); searched local with the original question | `test -n "$OPENROUTER_API_KEY"`; report the env var name, never its value |
|
|
238
258
|
| `query-decomposition-failed` | Decomposition model call failed; searched the original question | Check `queryDecomposition.model` resolves (`autorag models list --provider openrouter`) |
|
|
239
259
|
| `follow-up-check-fallback` | Jev post-fast-answer check unavailable; the run verified | Same as `query-route-fallback` |
|
|
@@ -168,14 +168,14 @@ AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite refresh --jso
|
|
|
168
168
|
MinSync and Jikji auto-install during refresh (never during a query). If they
|
|
169
169
|
are missing or broken, run a full refresh or return to setup rather than
|
|
170
170
|
silently degrading to lexical-only search.
|
|
171
|
-
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
171
|
+
- AutoRAG owns no embedding model; MinSync provisions
|
|
172
|
+
`Qwen/Qwen3-Embedding-0.6B` (1024 dimensions) in `~/.cache/minsync/models`. A
|
|
173
|
+
configured remote `minSync.embedder` is used first; otherwise a running
|
|
174
|
+
`minsync serve` daemon (started by `refresh`/`watch`, never by a search), and
|
|
175
|
+
failing that MinSync's in-process engine via the optional
|
|
176
|
+
`@nomadamas/minsync-native` addon. The local default needs no embedder flags
|
|
177
|
+
and no API key. Override the embedder only when intentionally using a remote
|
|
178
|
+
provider.
|
|
179
179
|
- Exact duplicate exclusion during refresh is enabled by default via the
|
|
180
180
|
external `dupey` CLI. Install dupey during setup when it is missing
|
|
181
181
|
(`command -v dupey || cargo install dupey --locked`) and tell the user the feature is
|
|
@@ -189,20 +189,28 @@ is impossible, refresh continues without this optimization and the user is
|
|
|
189
189
|
told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
|
|
190
190
|
index every copy.
|
|
191
191
|
|
|
192
|
-
|
|
193
|
-
(`
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
192
|
+
AutoRAG owns no embedding model. MinSync provisions the embedder
|
|
193
|
+
(`Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions) in its own cache
|
|
194
|
+
(`~/.cache/minsync/models`); there is no local model download and no bundled
|
|
195
|
+
runtime. One shared embedder serves everything that embeds inside
|
|
196
|
+
AutoRAG: MinSync indexing/query, memory similar-question search, the
|
|
197
|
+
chat-crawler sidecars, the GitHub Gist vector index, and the endpoint external
|
|
198
|
+
CLIs are pointed at. The default needs no embedder flags and no API key. It
|
|
199
|
+
resolves in order:
|
|
200
|
+
|
|
201
|
+
1. A configured remote endpoint (`minSync.embedder`), used for everything; no
|
|
202
|
+
daemon is started and no local model is loaded.
|
|
203
|
+
2. A running `minsync serve` daemon (loopback only; MinSync 0.4.11+).
|
|
204
|
+
AutoRAG never starts it: run `minsync serve --idle-timeout 0` yourself, as
|
|
205
|
+
a launchd/systemd service (see `docs/minsync-setup.md`), or press
|
|
206
|
+
Ctrl+Shift+E in `autorag tui`. `autorag setup` only reports whether it is
|
|
207
|
+
alive.
|
|
208
|
+
3. MinSync's in-process engine, through the optional Node-API addon
|
|
209
|
+
`@nomadamas/minsync-native` (same engine and weights, so bit-for-bit
|
|
210
|
+
identical vectors; not bundled). Without a daemon and without the addon,
|
|
211
|
+
embedding fails with an error naming the missing package.
|
|
212
|
+
|
|
213
|
+
Override the embedder only when intentionally using a remote provider:
|
|
206
214
|
|
|
207
215
|
```bash
|
|
208
216
|
autorag init \
|
|
@@ -213,9 +221,14 @@ autorag init \
|
|
|
213
221
|
--embedder-batch-size 64
|
|
214
222
|
```
|
|
215
223
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
224
|
+
A remote `minSync.embedder` names an `id` (`openai:<model>`, `tei:<model>`, or
|
|
225
|
+
any id together with a `baseUrl`) plus optional `apiKeyEnv` (the **name** of the
|
|
226
|
+
environment variable holding the bearer token), `dimension`, `batchSize`, and
|
|
227
|
+
`timeoutMs`. OpenAI-protocol endpoints POST `{baseUrl}/v1/embeddings`; a `tei:`
|
|
228
|
+
endpoint POSTs `{baseUrl}/embed`; `openai:` with no `baseUrl` means
|
|
229
|
+
`https://api.openai.com`. Only store the environment-variable name, never its
|
|
230
|
+
value. `dimension` and `batchSize` must be positive integers, and `dimension`
|
|
231
|
+
must match the endpoint (the local default is 1024).
|
|
219
232
|
|
|
220
233
|
### Jev routing and question decomposition (on by default)
|
|
221
234
|
|
|
@@ -336,7 +349,7 @@ keeps its mirrors and reports `search-root-unavailable`; see
|
|
|
336
349
|
|
|
337
350
|
## Probe and configure datasource skills (setup wizard)
|
|
338
351
|
|
|
339
|
-
Use `autorag setup` to probe the
|
|
352
|
+
Use `autorag setup` to probe the embedding backend, MinSync, and known
|
|
340
353
|
datasources automatically:
|
|
341
354
|
|
|
342
355
|
```bash
|
|
@@ -393,13 +406,15 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
|
|
|
393
406
|
Tokens are environment-variable names, not raw secrets. CLI-backed connectors
|
|
394
407
|
keep authentication in their external tool configuration.
|
|
395
408
|
|
|
396
|
-
Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.
|
|
409
|
+
Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.2.2` or newer)
|
|
397
410
|
and configured through its own Himalaya account. AutoRAG runs its local `sync`
|
|
398
411
|
and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
|
|
399
412
|
hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
|
|
400
|
-
fails with `text array must be non-empty`. 0.2.
|
|
401
|
-
|
|
402
|
-
|
|
413
|
+
fails with `text array must be non-empty`. 0.2.2 embeds through the same
|
|
414
|
+
backend as the rest of AutoRAG (a remote `minSync.embedder`, else the
|
|
415
|
+
`minsync serve` daemon) instead of its own in-process model, and keeps vectors
|
|
416
|
+
in LanceDB. Switching from the old native model rebuilds the semantic index on
|
|
417
|
+
the next `index`. kakao needs `lazykatok` 0.3.3 or newer for the same reason.
|
|
403
418
|
Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
|
|
404
419
|
datasource is removed.
|
|
405
420
|
|
|
@@ -415,7 +430,8 @@ autorag refresh --json
|
|
|
415
430
|
autorag search "summarize the collection" --top-k 3 --json --debug
|
|
416
431
|
```
|
|
417
432
|
|
|
418
|
-
- `setup` probes
|
|
433
|
+
- `setup` probes the embedding backend (`embedding.backend`), MinSync, and
|
|
434
|
+
datasource readiness.
|
|
419
435
|
- `status` is model-free and path-opaque.
|
|
420
436
|
- `health` resolves the single model, checks credential presence, and normally
|
|
421
437
|
performs one live completion probe.
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
|
2
|
-
import { Type } from "typebox";
|
|
3
|
-
export declare const RECOMMEND_PEER_TARGETS_TOOL_NAME = "recommend_peer_targets";
|
|
4
|
-
declare const recommendPeerTargetsSchema: Type.TObject<{
|
|
5
|
-
query: Type.TString;
|
|
6
|
-
}>;
|
|
7
|
-
export interface PeerTargetRecommendation {
|
|
8
|
-
readonly alias: string;
|
|
9
|
-
readonly matchedTerms: readonly string[];
|
|
10
|
-
readonly displayName?: string;
|
|
11
|
-
readonly description?: string;
|
|
12
|
-
}
|
|
13
|
-
export interface RecommendPeerTargetsDetails {
|
|
14
|
-
readonly method: "recommend_peer_targets";
|
|
15
|
-
readonly resultCount: number;
|
|
16
|
-
readonly matches: readonly PeerTargetRecommendation[];
|
|
17
|
-
}
|
|
18
|
-
export declare function createRecommendPeerTargetsTool(workspacePath: string): AgentTool<typeof recommendPeerTargetsSchema, RecommendPeerTargetsDetails>;
|
|
19
|
-
export {};
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
|
2
|
-
import { Type } from "typebox";
|
|
3
|
-
import { type SimplexQueryState } from "../p2p/simplex-query-store.js";
|
|
4
|
-
import type { SimplexTransport } from "../p2p/simplex-transport.js";
|
|
5
|
-
import type { PeerQueryResponse } from "../p2p/wire.js";
|
|
6
|
-
import { EvidenceLedger } from "./evidence-ledger.js";
|
|
7
|
-
export declare const QUERY_PEER_AGENT_TOOL_NAME = "query_peer_agent";
|
|
8
|
-
declare const queryPeerAgentSchema: Type.TObject<{
|
|
9
|
-
alias: Type.TString;
|
|
10
|
-
query: Type.TString;
|
|
11
|
-
topK: Type.TOptional<Type.TInteger>;
|
|
12
|
-
scope: Type.TOptional<Type.TString>;
|
|
13
|
-
}>;
|
|
14
|
-
export interface QueryPeerAgentResult {
|
|
15
|
-
readonly number: number;
|
|
16
|
-
readonly title: string;
|
|
17
|
-
readonly summary: string;
|
|
18
|
-
readonly source: string;
|
|
19
|
-
readonly excerpt: string;
|
|
20
|
-
}
|
|
21
|
-
export interface QueryPeerAgentDetails {
|
|
22
|
-
readonly method: "query_peer_agent";
|
|
23
|
-
readonly ok: boolean;
|
|
24
|
-
readonly alias: string;
|
|
25
|
-
readonly contactId?: number;
|
|
26
|
-
readonly pendingId?: string;
|
|
27
|
-
readonly expiresAt?: string;
|
|
28
|
-
readonly status?: "ok" | "rejected" | "pending";
|
|
29
|
-
readonly answer?: string;
|
|
30
|
-
readonly results?: readonly QueryPeerAgentResult[];
|
|
31
|
-
readonly diagnostics?: PeerQueryResponse["diagnostics"];
|
|
32
|
-
readonly fileCount?: number;
|
|
33
|
-
readonly message?: string;
|
|
34
|
-
}
|
|
35
|
-
export interface QueryPeerAgentToolOptions {
|
|
36
|
-
readonly workspacePath: string;
|
|
37
|
-
readonly openTransport: () => Promise<SimplexTransport>;
|
|
38
|
-
readonly fastTimeoutMs?: number;
|
|
39
|
-
readonly autoStart?: boolean;
|
|
40
|
-
readonly sessionId?: () => string | undefined;
|
|
41
|
-
readonly onResponse?: (state: SimplexQueryState, response: PeerQueryResponse) => void;
|
|
42
|
-
readonly onExpired?: (state: SimplexQueryState) => void;
|
|
43
|
-
/** Run evidence ledger; each peer result is registered so the final emit can cite it by id. */
|
|
44
|
-
readonly ledger?: EvidenceLedger;
|
|
45
|
-
}
|
|
46
|
-
export interface QueryPeerAgentTool extends AgentTool<typeof queryPeerAgentSchema, QueryPeerAgentDetails> {
|
|
47
|
-
readonly ready: Promise<void>;
|
|
48
|
-
close(): Promise<void>;
|
|
49
|
-
}
|
|
50
|
-
export declare function createQueryPeerAgentTool(options: QueryPeerAgentToolOptions): QueryPeerAgentTool;
|
|
51
|
-
export {};
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { type RuntimeStatus } from "../../embedding-runtime/index.js";
|
|
2
|
-
import type { CommandContext } from "./types.js";
|
|
3
|
-
export interface GatewayCommandDeps {
|
|
4
|
-
runtimeStatus?: () => Promise<RuntimeStatus>;
|
|
5
|
-
stopRuntime?: () => Promise<void>;
|
|
6
|
-
}
|
|
7
|
-
export declare function runGateway(ctx: CommandContext, deps?: GatewayCommandDeps): Promise<number>;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { CommandContext } from "./types.js";
|
|
2
|
-
/** Resolve the workspace used for P2P identity/policy, matching `autorag serve`. */
|
|
3
|
-
export declare function resolveP2pWorkspace(ctx: CommandContext): string;
|
|
4
|
-
/**
|
|
5
|
-
* `autorag p2p` — SimpleX peer trust management.
|
|
6
|
-
*/
|
|
7
|
-
export declare function runP2p(ctx: CommandContext): Promise<number>;
|