@autorag/librarian 2.10.5 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +14 -9
- package/README.md +65 -25
- package/dist/agent/agent.d.ts +51 -49
- package/dist/agent/answer-guidelines.d.ts +22 -1
- package/dist/agent/context-token-guard.d.ts +11 -0
- package/dist/agent/discord-remote.d.ts +30 -0
- package/dist/agent/embedding-controls.d.ts +22 -0
- package/dist/agent/index.d.ts +1 -2
- package/dist/agent/mcp-client.d.ts +27 -0
- package/dist/agent/pi-session.d.ts +8 -1
- package/dist/agent/query-decomposition.d.ts +1 -1
- package/dist/agent/search-documents.d.ts +2 -2
- package/dist/agent/search-progress.d.ts +96 -0
- package/dist/agent/system-prompt.d.ts +1 -1
- package/dist/agent/welcome-header.d.ts +35 -0
- package/dist/cli/commands/models.d.ts +0 -14
- package/dist/cli/commands/search.d.ts +3 -4
- package/dist/cli/commands/setup.d.ts +2 -0
- package/dist/cli/config.d.ts +7 -19
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +16364 -18337
- package/dist/cli/setup.d.ts +22 -36
- package/dist/cloud/config.d.ts +3 -3
- package/dist/core.js +7372 -9206
- package/dist/datasource/embedding-env.d.ts +29 -0
- package/dist/datasource/errors.d.ts +1 -1
- package/dist/datasource/index.d.ts +2 -1
- package/dist/datasource/sidecar/config.d.ts +3 -0
- package/dist/datasource/sidecar/index.d.ts +1 -1
- package/dist/datasource/skills/discrawl/client.d.ts +1 -0
- package/dist/datasource/skills/discrawl/embedding.d.ts +17 -0
- package/dist/datasource/skills/discrawl/types.d.ts +11 -3
- package/dist/datasource/skills/factory.d.ts +17 -1
- package/dist/datasource/skills/github-gist/semantic.d.ts +5 -5
- package/dist/datasource/skills/github-gist/skill.d.ts +2 -3
- package/dist/datasource/skills/hister/client.d.ts +83 -0
- package/dist/datasource/skills/hister/embedding.d.ts +64 -0
- package/dist/datasource/skills/hister/index.d.ts +2 -0
- package/dist/datasource/skills/hister/setup.d.ts +87 -0
- package/dist/datasource/skills/hister/skill.d.ts +63 -0
- package/dist/datasource/skills/lazykatok/types.d.ts +9 -1
- package/dist/datasource/skills/mailcrawl/types.d.ts +9 -1
- package/dist/datasource/skills/obsidian/index.d.ts +4 -6
- package/dist/datasource/skills/obsidian/methods.d.ts +13 -17
- package/dist/datasource/skills/obsidian/paths.d.ts +24 -24
- package/dist/datasource/skills/obsidian/skill.d.ts +10 -15
- package/dist/datasource/skills/obsidian/staging.d.ts +50 -0
- package/dist/datasource/skills/obsidian/vault-index.d.ts +44 -0
- package/dist/embedding/daemon.d.ts +57 -0
- package/dist/embedding/embedder.d.ts +59 -0
- package/dist/embedding/http-embedder.d.ts +28 -0
- package/dist/embedding/in-process.d.ts +19 -0
- package/dist/embedding/index.d.ts +28 -0
- package/dist/embedding/managed.d.ts +44 -0
- package/dist/embedding/prefixes.d.ts +18 -0
- package/dist/embedding/settings.d.ts +9 -0
- package/dist/index.js +7465 -9298
- package/dist/mcp/index.js +7460 -9294
- package/dist/memory/context.d.ts +1 -1
- package/dist/memory/similar-queries.d.ts +1 -1
- package/dist/minsync/binary.d.ts +21 -0
- package/dist/minsync/client.d.ts +11 -10
- package/dist/minsync/index.d.ts +1 -1
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/method.d.ts +3 -11
- package/dist/minsync/process.d.ts +1 -0
- package/dist/minsync/types.d.ts +0 -2
- package/dist/minsync/workspace-lock.d.ts +7 -0
- package/dist/process/terminate-tree.d.ts +4 -5
- package/dist/retrieval/rerank.d.ts +10 -3
- package/licenses/hister-AGPL-3.0-notice.txt +47 -0
- package/package.json +4 -4
- package/skills/autorag/SKILL.md +9 -11
- package/skills/autorag-doctor/SKILL.md +50 -29
- package/skills/autorag-lite-setup/SKILL.md +31 -13
- package/skills/autorag-setup/SKILL.md +99 -26
- package/dist/agent/peer-target-tool.d.ts +0 -19
- package/dist/agent/query-peer-tool.d.ts +0 -51
- package/dist/cli/commands/gateway.d.ts +0 -7
- package/dist/cli/commands/p2p-policy.d.ts +0 -2
- package/dist/cli/commands/p2p.d.ts +0 -7
- package/dist/cli/commands/serve.d.ts +0 -19
- package/dist/datasource/skills/obsidian/client.d.ts +0 -15
- package/dist/datasource/skills/obsidian/types.d.ts +0 -92
- package/dist/embedding-runtime/cache.d.ts +0 -30
- package/dist/embedding-runtime/gateway-embedder.d.ts +0 -41
- package/dist/embedding-runtime/gateway.d.ts +0 -27
- package/dist/embedding-runtime/index.d.ts +0 -120
- package/dist/embedding-runtime/manifest.d.ts +0 -20
- package/dist/embedding-runtime/supervisor.d.ts +0 -74
- package/dist/embedding-runtime/types.d.ts +0 -70
- package/dist/p2p/approval-store.d.ts +0 -32
- package/dist/p2p/egress-gate.d.ts +0 -22
- package/dist/p2p/file-sharing.d.ts +0 -33
- package/dist/p2p/injection-classifier.d.ts +0 -33
- package/dist/p2p/injection-gate.d.ts +0 -37
- package/dist/p2p/pii-gate.d.ts +0 -47
- package/dist/p2p/policy-filter.d.ts +0 -15
- package/dist/p2p/policy.d.ts +0 -77
- package/dist/p2p/simplex-query-store.d.ts +0 -44
- package/dist/p2p/simplex-server.d.ts +0 -115
- package/dist/p2p/simplex-transport.d.ts +0 -62
- package/dist/p2p/wire.d.ts +0 -39
- package/licenses/embedding-assets.json +0 -85
- package/licenses/llama.cpp-MIT.txt +0 -21
- package/licenses/qwen3-embedding-notice.txt +0 -13
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Path helpers for the Obsidian
|
|
2
|
+
* Path helpers for the Obsidian datasource.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* `<workspaceRoot>/.autorag/datasources/obsidian/<
|
|
6
|
-
* `
|
|
7
|
-
*
|
|
8
|
-
* the form `/obsidian/<instance-
|
|
4
|
+
* Everything AutoRAG owns for one vault lives under
|
|
5
|
+
* `<workspaceRoot>/.autorag/datasources/obsidian/<instance>/`: a MinSync workspace of its own
|
|
6
|
+
* (`minsync/`, never shared with the local-file index) and the staging record (`staging.json`)
|
|
7
|
+
* that maps staged notes back to the vault. Callers see slash-hierarchical opaque sources of
|
|
8
|
+
* the form `/obsidian/<instance>/<vault-relative note path>`.
|
|
9
9
|
*/
|
|
10
|
-
export declare const
|
|
11
|
-
export declare const
|
|
12
|
-
export declare
|
|
13
|
-
|
|
14
|
-
export declare function
|
|
15
|
-
/**
|
|
16
|
-
export declare function
|
|
17
|
-
/**
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
export declare function
|
|
26
|
-
/**
|
|
27
|
-
export declare function
|
|
10
|
+
export declare const OBSIDIAN_DATASOURCE_ID = "obsidian";
|
|
11
|
+
export declare const OBSIDIAN_SKILL_TYPE = "obsidian-vault";
|
|
12
|
+
export declare function obsidianDatasourceDir(workspaceRoot: string, instanceId: string): string;
|
|
13
|
+
/** The MinSync workspace this vault is indexed in; staged notes live in its `files/` folder. */
|
|
14
|
+
export declare function obsidianMinSyncWorkspace(workspaceRoot: string, instanceId: string): string;
|
|
15
|
+
/** Kept beside the MinSync workspace, outside the tree MinSync indexes. */
|
|
16
|
+
export declare function obsidianStagingStatePath(workspaceRoot: string, instanceId: string): string;
|
|
17
|
+
/**
|
|
18
|
+
* Escape what would change the meaning of a source segment: `%` (the escape itself), `/` (the
|
|
19
|
+
* separator, possible in an instance id), `#` (datasource sources never carry a fragment), `\`
|
|
20
|
+
* (not valid in a virtual path) and `*` (a scope glob). Everything else, spaces and non-Latin
|
|
21
|
+
* letters included, stays readable.
|
|
22
|
+
*/
|
|
23
|
+
export declare function encodeSourceSegment(segment: string): string;
|
|
24
|
+
/** The root source of one vault: the scope that selects every note in it. */
|
|
25
|
+
export declare function obsidianInstanceSource(instanceId: string): string;
|
|
26
|
+
/** The source of one note, from its vault-relative path written with `/` separators. */
|
|
27
|
+
export declare function obsidianNoteSource(instanceId: string, relativePath: string): string;
|
|
@@ -1,37 +1,33 @@
|
|
|
1
1
|
import type { RetrievalMethod } from "../../../retrieval/types.js";
|
|
2
2
|
import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
|
|
3
|
-
import { type
|
|
4
|
-
import type { QmdEmbedResult, QmdEnsureResult, QmdUpdateResult } from "./types.js";
|
|
5
|
-
export interface ObsidianSkillClient extends ObsidianSearchClient {
|
|
6
|
-
ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
|
|
7
|
-
update(signal?: AbortSignal): Promise<QmdUpdateResult>;
|
|
8
|
-
embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
|
|
9
|
-
}
|
|
3
|
+
import { type ObsidianMinSyncOptions } from "./vault-index.js";
|
|
10
4
|
export interface ObsidianSkillOptions {
|
|
11
5
|
readonly datasourceId?: string;
|
|
12
|
-
readonly client?: ObsidianSkillClient;
|
|
13
6
|
readonly vaultPath?: string;
|
|
14
|
-
readonly binaryPath?: string;
|
|
15
7
|
readonly instanceId?: string;
|
|
16
8
|
readonly instances?: readonly string[];
|
|
17
9
|
readonly pollingIntervalMs?: number;
|
|
18
10
|
readonly tags?: readonly string[];
|
|
19
11
|
readonly lastIndexedAt?: number;
|
|
12
|
+
/** Where `.autorag/datasources/obsidian/<instance>` is kept. Required to index or search. */
|
|
20
13
|
readonly workspaceRoot?: string;
|
|
21
|
-
|
|
22
|
-
readonly
|
|
23
|
-
readonly configPath?: string;
|
|
14
|
+
/** The install's MinSync binary, embedder and chunking, shared with the local-file index. */
|
|
15
|
+
readonly minSync?: ObsidianMinSyncOptions;
|
|
24
16
|
readonly connectorOptions?: {
|
|
25
17
|
readonly vaultPath?: string;
|
|
26
18
|
};
|
|
27
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* An Obsidian vault as a datasource: its markdown notes are indexed by MinSync (BM25 and
|
|
22
|
+
* semantic) in a workspace of their own, and a note's source is its vault-relative path, so a
|
|
23
|
+
* folder is an ordinary `scope`.
|
|
24
|
+
*/
|
|
28
25
|
export declare class ObsidianSkill implements DatasourceSkill {
|
|
29
|
-
private readonly
|
|
26
|
+
private readonly vaultIndex;
|
|
30
27
|
private readonly instanceId;
|
|
31
28
|
private readonly instances;
|
|
32
29
|
private readonly pollingIntervalMs;
|
|
33
30
|
private readonly tags;
|
|
34
|
-
private readonly vaultPath;
|
|
35
31
|
private lastIndexedAt;
|
|
36
32
|
private lastError;
|
|
37
33
|
constructor(options?: ObsidianSkillOptions);
|
|
@@ -41,5 +37,4 @@ export declare class ObsidianSkill implements DatasourceSkill {
|
|
|
41
37
|
retrievalMethods(): readonly RetrievalMethod[];
|
|
42
38
|
describeSources(): readonly SourceDescription[];
|
|
43
39
|
skillManifest(): DatasourceSkillManifest;
|
|
44
|
-
private fail;
|
|
45
40
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export interface VaultNote {
|
|
2
|
+
/** Vault-relative path, NFC-normalized, `/`-separated. */
|
|
3
|
+
readonly relativePath: string;
|
|
4
|
+
readonly sourcePath: string;
|
|
5
|
+
readonly size: number;
|
|
6
|
+
/** Exact nanosecond mtime as a decimal string (it exceeds a double's integer range). */
|
|
7
|
+
readonly mtimeNs: string;
|
|
8
|
+
}
|
|
9
|
+
export interface VaultListing {
|
|
10
|
+
readonly notes: ReadonlyMap<string, VaultNote>;
|
|
11
|
+
/** Folders (vault-relative) that could not be read; their staged notes are kept as they were. */
|
|
12
|
+
readonly unreadableFolders: readonly string[];
|
|
13
|
+
/** Notes dropped because another note has the same NFC-normalized path. */
|
|
14
|
+
readonly collisions: readonly string[];
|
|
15
|
+
}
|
|
16
|
+
export declare class VaultUnavailableError extends Error {
|
|
17
|
+
readonly code: string;
|
|
18
|
+
constructor(code: string);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* List the vault's notes. Dot-prefixed folders and files (`.obsidian`, `.trash`, `.git`),
|
|
22
|
+
* `node_modules`, and symlinks are not part of the vault's knowledge and are skipped.
|
|
23
|
+
* Throws {@link VaultUnavailableError} when the vault root itself cannot be read.
|
|
24
|
+
*/
|
|
25
|
+
export declare function listVaultNotes(vaultPath: string): Promise<VaultListing>;
|
|
26
|
+
interface StagingEntry {
|
|
27
|
+
readonly sourcePath: string;
|
|
28
|
+
readonly size: number;
|
|
29
|
+
readonly mtimeNs: string;
|
|
30
|
+
}
|
|
31
|
+
export interface StagingState {
|
|
32
|
+
readonly version: 1;
|
|
33
|
+
readonly entries: Readonly<Record<string, StagingEntry>>;
|
|
34
|
+
}
|
|
35
|
+
export declare function loadStagingState(statePath: string): StagingState;
|
|
36
|
+
export interface StageResult {
|
|
37
|
+
readonly staged: number;
|
|
38
|
+
readonly copied: number;
|
|
39
|
+
readonly removed: number;
|
|
40
|
+
/** Notes that could not be copied this time; an earlier staged copy, if any, was kept. */
|
|
41
|
+
readonly failed: readonly string[];
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Bring `<filesRoot>` in line with the vault: copy new and changed notes, delete notes that no
|
|
45
|
+
* longer exist, and leave notes under an unreadable folder exactly as they were.
|
|
46
|
+
*/
|
|
47
|
+
export declare function stageNotes(filesRoot: string, statePath: string, listing: VaultListing, options?: {
|
|
48
|
+
readonly force?: boolean;
|
|
49
|
+
}): StageResult;
|
|
50
|
+
export {};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { MinSyncVectorMethodOptions } from "../../../minsync/method.js";
|
|
2
|
+
import type { RetrievalResult } from "../../../retrieval/types.js";
|
|
3
|
+
import type { DatasourceDiagnostic, DatasourceDiagnosticCode } from "../../types.js";
|
|
4
|
+
/** The MinSync settings a vault shares with the rest of the install: binary, embedder, chunking. */
|
|
5
|
+
export type ObsidianMinSyncOptions = Pick<MinSyncVectorMethodOptions, "binaryPath" | "autoInstall" | "installer" | "embedder" | "maxChunkSize">;
|
|
6
|
+
export interface ObsidianVaultIndexOptions {
|
|
7
|
+
readonly instanceId: string;
|
|
8
|
+
readonly vaultPath: string | undefined;
|
|
9
|
+
readonly workspaceRoot: string | undefined;
|
|
10
|
+
readonly minSync?: ObsidianMinSyncOptions;
|
|
11
|
+
}
|
|
12
|
+
export type ObsidianSearchMode = "bm25" | "semantic";
|
|
13
|
+
export interface ObsidianSearchOptions {
|
|
14
|
+
readonly topK?: number;
|
|
15
|
+
readonly scope?: string;
|
|
16
|
+
readonly signal?: AbortSignal;
|
|
17
|
+
}
|
|
18
|
+
export type VaultSyncOutcome = {
|
|
19
|
+
readonly ok: true;
|
|
20
|
+
readonly notes: number;
|
|
21
|
+
readonly diagnostics: readonly DatasourceDiagnostic[];
|
|
22
|
+
} | {
|
|
23
|
+
readonly ok: false;
|
|
24
|
+
readonly code: DatasourceDiagnosticCode;
|
|
25
|
+
readonly message: string;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* One vault, indexed by MinSync in a workspace of its own. Notes are staged under the workspace's
|
|
29
|
+
* `files/` folder (see `staging.ts`), `minsync sync` embeds and indexes them incrementally, and a
|
|
30
|
+
* query runs against that index and maps each chunk back to its note.
|
|
31
|
+
*/
|
|
32
|
+
export declare class ObsidianVaultIndex {
|
|
33
|
+
private readonly instanceId;
|
|
34
|
+
private readonly vaultPath;
|
|
35
|
+
private readonly workspaceRoot;
|
|
36
|
+
private readonly minSync;
|
|
37
|
+
constructor(options: ObsidianVaultIndexOptions);
|
|
38
|
+
/** Mirror the vault's notes into the MinSync workspace and sync it. Never throws except on abort. */
|
|
39
|
+
sync(signal?: AbortSignal): Promise<VaultSyncOutcome>;
|
|
40
|
+
private syncLocked;
|
|
41
|
+
/** Search the indexed vault. Rejects, with the reason, when the vault was never indexed or a sync is running. */
|
|
42
|
+
query(mode: ObsidianSearchMode, text: string, options?: ObsidianSearchOptions): Promise<RetrievalResult[]>;
|
|
43
|
+
private queryLocked;
|
|
44
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/** The model `minsync serve` serves by default; callers pass another to look for that one. */
|
|
2
|
+
export declare const DEFAULT_DAEMON_MODEL = "Qwen/Qwen3-Embedding-0.6B";
|
|
3
|
+
export interface MinSyncDaemon {
|
|
4
|
+
/** `http://127.0.0.1:<port>`; the OpenAI route lives under `/v1`, TEI at `/embed`. */
|
|
5
|
+
readonly baseUrl: string;
|
|
6
|
+
readonly model: string;
|
|
7
|
+
readonly dimension: number;
|
|
8
|
+
readonly instance: string;
|
|
9
|
+
/** True when {@link ensureMinSyncDaemon} spawned it. */
|
|
10
|
+
readonly started: boolean;
|
|
11
|
+
readonly pid?: number;
|
|
12
|
+
}
|
|
13
|
+
export declare class MinSyncDaemonError extends Error {
|
|
14
|
+
constructor(message: string, options?: ErrorOptions);
|
|
15
|
+
}
|
|
16
|
+
export interface ProbeOptions {
|
|
17
|
+
/** User home; MinSync keeps its server state under `<home>/.cache/minsync/serve`. */
|
|
18
|
+
readonly home?: string;
|
|
19
|
+
/** Only accept a server of this model. Defaults to {@link DEFAULT_DAEMON_MODEL}. */
|
|
20
|
+
readonly model?: string;
|
|
21
|
+
readonly fetchImpl?: typeof fetch;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* MinSync publishes `host`/`port` (and `pid`) per model instance under
|
|
25
|
+
* `~/.cache/minsync/serve/<instance>/` and removes them on a clean stop. A leftover file from a crashed
|
|
26
|
+
* server is harmless because every candidate must answer `GET /health` as a loopback `status: ok` server of
|
|
27
|
+
* the wanted model before it is used.
|
|
28
|
+
*/
|
|
29
|
+
export declare function minSyncServeStateRoot(home?: string): string;
|
|
30
|
+
export declare function probeMinSyncDaemon(options?: ProbeOptions): Promise<MinSyncDaemon | undefined>;
|
|
31
|
+
export interface EnsureDaemonOptions extends ProbeOptions {
|
|
32
|
+
/** The `minsync` executable. Required only when no daemon is running yet. */
|
|
33
|
+
readonly binaryPath: string;
|
|
34
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
35
|
+
/** Working directory of the spawned server. `minsync serve` reads `.minsync/config.toml` from it. */
|
|
36
|
+
readonly cwd?: string;
|
|
37
|
+
/** How long to wait for the server to publish and answer; the first start may download the model. */
|
|
38
|
+
readonly readyTimeoutMs?: number;
|
|
39
|
+
/** Idle seconds before the daemon exits and frees the model (`minsync serve --idle-timeout`). */
|
|
40
|
+
readonly idleTimeoutSeconds?: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The running `minsync serve` daemon, starting one detached when none answers. The daemon is MinSync's:
|
|
44
|
+
* it owns the model weights, the cache, and its own idle shutdown. Concurrent starters are safe because
|
|
45
|
+
* `minsync serve` adopts an already-running server of the same model instead of loading a second copy.
|
|
46
|
+
*/
|
|
47
|
+
export declare function ensureMinSyncDaemon(options: EnsureDaemonOptions): Promise<MinSyncDaemon>;
|
|
48
|
+
export interface StopDaemonOptions extends ProbeOptions {
|
|
49
|
+
/** How long to wait for the server to remove its discovery files and stop answering. */
|
|
50
|
+
readonly timeoutMs?: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Stops the running `minsync serve` daemon with SIGTERM, which `minsync serve` handles by removing its
|
|
54
|
+
* discovery files and exiting. The pid comes from the server's own `GET /health` answer, so only a live
|
|
55
|
+
* loopback server of the wanted model is ever signalled. Returns the stopped daemon, or undefined when none ran.
|
|
56
|
+
*/
|
|
57
|
+
export declare function stopMinSyncDaemon(options?: StopDaemonOptions): Promise<MinSyncDaemon | undefined>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The contract every semantic consumer in AutoRAG codes against. AutoRAG does not run an embedding model:
|
|
3
|
+
* vectors come from MinSync (the `minsync serve` daemon, or MinSync's in-process engine) or from a
|
|
4
|
+
* remote endpoint the operator configured.
|
|
5
|
+
*/
|
|
6
|
+
export interface EmbeddingIdentity {
|
|
7
|
+
readonly provider: string;
|
|
8
|
+
readonly model: string;
|
|
9
|
+
readonly dimension: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* What a text is for. Instruction-aware models embed a search query differently from the document it should
|
|
13
|
+
* match, so a caller says which side it is on. `"document"` is the default.
|
|
14
|
+
*/
|
|
15
|
+
export type EmbedRole = "query" | "document";
|
|
16
|
+
export interface EmbedOptions {
|
|
17
|
+
readonly role?: EmbedRole;
|
|
18
|
+
}
|
|
19
|
+
export interface Embedder {
|
|
20
|
+
identity(): Promise<EmbeddingIdentity>;
|
|
21
|
+
embed(texts: readonly string[], options?: EmbedOptions): Promise<readonly (readonly number[])[]>;
|
|
22
|
+
}
|
|
23
|
+
/** Where a {@link ManagedEmbedder} is sending text right now. `none` means nothing can embed. */
|
|
24
|
+
export type EmbeddingBackend = "daemon" | "in-process" | "remote" | "none";
|
|
25
|
+
/** Whether the embedding backend answers right now. Probed on every call; nothing is cached. */
|
|
26
|
+
export interface EmbedderHealth {
|
|
27
|
+
readonly backend: EmbeddingBackend;
|
|
28
|
+
readonly alive: boolean;
|
|
29
|
+
/** The answering server's address when alive, or why it is not. */
|
|
30
|
+
readonly detail?: string;
|
|
31
|
+
}
|
|
32
|
+
/** Where a {@link ManagedEmbedder} currently sends text, for external CLIs that need a URL. */
|
|
33
|
+
export interface EmbedderEndpoint {
|
|
34
|
+
/** Server root; the OpenAI route is under `/v1`, TEI at `/embed`. */
|
|
35
|
+
readonly baseUrl: string;
|
|
36
|
+
readonly protocol: "openai" | "tei";
|
|
37
|
+
readonly model: string;
|
|
38
|
+
readonly dimension: number | undefined;
|
|
39
|
+
/** Name of the environment variable that holds the bearer token; remote endpoints only. */
|
|
40
|
+
readonly apiKeyEnv?: string;
|
|
41
|
+
/**
|
|
42
|
+
* Prefixes this endpoint's consumers must apply themselves: the server embeds text as sent. Set only when
|
|
43
|
+
* the embedder was given prefixes.
|
|
44
|
+
*/
|
|
45
|
+
readonly queryPrefix?: string;
|
|
46
|
+
readonly passagePrefix?: string;
|
|
47
|
+
readonly backend: "daemon" | "remote";
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* An {@link Embedder} that reports where it embeds. AutoRAG never starts or stops a server: the operator
|
|
51
|
+
* runs `minsync serve` or configures a remote endpoint, and `health` / `describeEndpoint` only look.
|
|
52
|
+
*/
|
|
53
|
+
export interface ManagedEmbedder extends Embedder {
|
|
54
|
+
/** Probes the backend now: the daemon's or the remote's `GET /health`. Never starts anything. */
|
|
55
|
+
health(): Promise<EmbedderHealth>;
|
|
56
|
+
/** The HTTP endpoint behind this embedder right now, or undefined while it would embed in-process. */
|
|
57
|
+
describeEndpoint(): Promise<EmbedderEndpoint | undefined>;
|
|
58
|
+
close(): Promise<void>;
|
|
59
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Embedder } from "./embedder.js";
|
|
2
|
+
export type EmbeddingProtocol = "openai" | "tei";
|
|
3
|
+
export interface HttpEmbedderOptions {
|
|
4
|
+
readonly protocol: EmbeddingProtocol;
|
|
5
|
+
/** Server root, with or without a trailing `/v1` (openai) and trailing slashes. */
|
|
6
|
+
readonly baseUrl: string;
|
|
7
|
+
readonly model: string;
|
|
8
|
+
/** Expected vector length. Learned from the first response when omitted. */
|
|
9
|
+
readonly dimension?: number;
|
|
10
|
+
/** Reported as `identity().provider`; stores keyed on identity re-embed when it changes. */
|
|
11
|
+
readonly provider: string;
|
|
12
|
+
/** Name of the environment variable holding the bearer token. Read on every call, never stored. */
|
|
13
|
+
readonly apiKeyEnv?: string;
|
|
14
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
15
|
+
/** Inputs per request. */
|
|
16
|
+
readonly batchSize?: number;
|
|
17
|
+
/** TEI only: ask the server to truncate inputs over its context instead of answering 413. */
|
|
18
|
+
readonly truncate?: boolean;
|
|
19
|
+
readonly timeoutMs?: number;
|
|
20
|
+
readonly fetchImpl?: typeof fetch;
|
|
21
|
+
}
|
|
22
|
+
/** The daemon rejects bodies over 32 MiB and more than 512 inputs; remote providers cap lower. */
|
|
23
|
+
export declare const DEFAULT_EMBED_BATCH_SIZE = 32;
|
|
24
|
+
/**
|
|
25
|
+
* Embedder over an HTTP endpoint: a `minsync serve` daemon (either route) or any remote OpenAI-compatible
|
|
26
|
+
* or TEI server. It owns no model and starts nothing.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createHttpEmbedder(options: HttpEmbedderOptions): Embedder;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Embedder } from "./embedder.js";
|
|
2
|
+
/** MinSync's Node-API addon: the same engine, weights and cache as `minsync`'s `native:` embedder. */
|
|
3
|
+
export declare const NATIVE_ADDON_PACKAGE = "@nomadamas/minsync-native";
|
|
4
|
+
export declare const DEFAULT_NATIVE_MODEL_ID = "native:Qwen/Qwen3-Embedding-0.6B";
|
|
5
|
+
export interface InProcessEmbedder extends Embedder {
|
|
6
|
+
/** Releases the model; later calls reject. */
|
|
7
|
+
close(): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export declare class NativeAddonMissingError extends Error {
|
|
10
|
+
constructor(cause: unknown);
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Loads MinSync's in-process embedder. AutoRAG ships no model and no inference runtime; the addon is an
|
|
14
|
+
* optional peer that brings MinSync's engine into this process, and MinSync provisions the weights into its
|
|
15
|
+
* own cache on first use.
|
|
16
|
+
*/
|
|
17
|
+
export declare function loadNativeEmbedder(options?: {
|
|
18
|
+
readonly id?: string;
|
|
19
|
+
}): Promise<InProcessEmbedder>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { MinSyncEmbedderConfig } from "../minsync/types.js";
|
|
2
|
+
import type { ManagedEmbedder } from "./embedder.js";
|
|
3
|
+
export { DEFAULT_DAEMON_MODEL, ensureMinSyncDaemon, MinSyncDaemonError, probeMinSyncDaemon, stopMinSyncDaemon, } from "./daemon.js";
|
|
4
|
+
export type { Embedder, EmbedderEndpoint, EmbedderHealth, EmbeddingBackend, EmbeddingIdentity, EmbedOptions, EmbedRole, ManagedEmbedder, } from "./embedder.js";
|
|
5
|
+
export { createHttpEmbedder, type EmbeddingProtocol } from "./http-embedder.js";
|
|
6
|
+
export { loadNativeEmbedder, NATIVE_ADDON_PACKAGE, NativeAddonMissingError } from "./in-process.js";
|
|
7
|
+
export { createManagedEmbedder, type ManagedEmbedderOptions, type RemoteEmbedderSettings } from "./managed.js";
|
|
8
|
+
export { type EmbeddingPrefixes, QWEN3_QUERY_PREFIX, resolveEmbeddingPrefixes } from "./prefixes.js";
|
|
9
|
+
export { remoteEmbedderFromMinSync } from "./settings.js";
|
|
10
|
+
export interface EmbedderFromConfigOptions {
|
|
11
|
+
/** The `minSync.embedder` block: it selects the embedding backend for MinSync and every other consumer. */
|
|
12
|
+
readonly embedder?: MinSyncEmbedderConfig;
|
|
13
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
14
|
+
/** User home that holds MinSync's `minsync serve` state; defaults to the real home. */
|
|
15
|
+
readonly home?: string;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* The single embedder AutoRAG shares between its own semantic stores and the datasources: a configured
|
|
19
|
+
* remote endpoint, else the `minsync serve` daemon the operator runs, else MinSync's in-process engine.
|
|
20
|
+
* AutoRAG never starts the daemon for it.
|
|
21
|
+
*/
|
|
22
|
+
export declare function createEmbedderFromConfig(options: EmbedderFromConfigOptions): ManagedEmbedder;
|
|
23
|
+
/**
|
|
24
|
+
* Unload every in-process model this process loaded. A short-lived command calls this on the way out so the
|
|
25
|
+
* addon's native threads cannot keep the process alive after its answer is written. A `minsync serve` daemon
|
|
26
|
+
* is not touched: it belongs to the operator.
|
|
27
|
+
*/
|
|
28
|
+
export declare function releaseEmbedders(): Promise<void>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ManagedEmbedder } from "./embedder.js";
|
|
2
|
+
import { type EmbeddingProtocol } from "./http-embedder.js";
|
|
3
|
+
import { type InProcessEmbedder } from "./in-process.js";
|
|
4
|
+
import type { EmbeddingPrefixes } from "./prefixes.js";
|
|
5
|
+
/** An operator-configured remote endpoint. When present, AutoRAG never looks for MinSync's daemon. */
|
|
6
|
+
export interface RemoteEmbedderSettings {
|
|
7
|
+
readonly baseUrl: string;
|
|
8
|
+
readonly model: string;
|
|
9
|
+
readonly protocol: EmbeddingProtocol;
|
|
10
|
+
readonly dimension?: number;
|
|
11
|
+
readonly apiKeyEnv?: string;
|
|
12
|
+
readonly batchSize?: number;
|
|
13
|
+
readonly timeoutMs?: number;
|
|
14
|
+
/** Reported as the identity provider; defaults to the protocol name. */
|
|
15
|
+
readonly provider?: string;
|
|
16
|
+
}
|
|
17
|
+
export interface ManagedEmbedderOptions {
|
|
18
|
+
readonly remote?: RemoteEmbedderSettings;
|
|
19
|
+
/**
|
|
20
|
+
* Prepended to every input by role: `query` for `role: "query"`, `passage` otherwise. Applied here, on the
|
|
21
|
+
* client, for every backend, because neither the daemon nor a remote server is asked to prefix.
|
|
22
|
+
*/
|
|
23
|
+
readonly prefixes?: EmbeddingPrefixes;
|
|
24
|
+
/** User home that holds MinSync's server state. */
|
|
25
|
+
readonly home?: string;
|
|
26
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
27
|
+
/** Model of the daemon to use. Defaults to MinSync's default Qwen3 embedder. */
|
|
28
|
+
readonly model?: string;
|
|
29
|
+
/** Loads MinSync's in-process engine; only called when no daemon answers. */
|
|
30
|
+
readonly inProcess?: () => Promise<InProcessEmbedder>;
|
|
31
|
+
readonly fetchImpl?: typeof fetch;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The one embedder every AutoRAG consumer shares. AutoRAG owns no embedding model and never starts or stops
|
|
35
|
+
* a process: the operator runs `minsync serve` (see docs/minsync-setup.md) or points at a remote endpoint.
|
|
36
|
+
*
|
|
37
|
+
* - **remote**: a configured endpoint is used as is; nothing local is looked for or loaded.
|
|
38
|
+
* - **daemon**: a running `minsync serve` is preferred, because it keeps one warm copy of the model for every
|
|
39
|
+
* process. It is looked up again on every call, so one started or stopped while AutoRAG runs is noticed.
|
|
40
|
+
* - **in-process**: MinSync's own engine, used when no daemon answers. Daemon and in-process vectors are
|
|
41
|
+
* identical for the same model and device, so both report one identity and stores never re-embed
|
|
42
|
+
* because the daemon came or went.
|
|
43
|
+
*/
|
|
44
|
+
export declare function createManagedEmbedder(options: ManagedEmbedderOptions): ManagedEmbedder;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { MinSyncEmbedderConfig } from "../minsync/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Qwen3-Embedding is instruction-aware: the query side is `Instruct: {task}\nQuery:{query}` and documents are
|
|
4
|
+
* embedded raw (model card `get_detailed_instruct`; there is no space after `Query:`). The task stays in
|
|
5
|
+
* English even for non-English queries, as the card recommends.
|
|
6
|
+
*/
|
|
7
|
+
export declare const QWEN3_QUERY_PREFIX = "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:";
|
|
8
|
+
/** Text prepended to an embedding input, by what the input is for. */
|
|
9
|
+
export interface EmbeddingPrefixes {
|
|
10
|
+
readonly query: string;
|
|
11
|
+
readonly passage: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The query and passage prefixes for the embedder `minSync.embedder` describes. An operator value, including
|
|
15
|
+
* `""`, always wins. Without one, MinSync's local Qwen3 gets the model's own query instruction; every other
|
|
16
|
+
* model gets none, because its prefix scheme is unknown here.
|
|
17
|
+
*/
|
|
18
|
+
export declare function resolveEmbeddingPrefixes(embedder: MinSyncEmbedderConfig | undefined): EmbeddingPrefixes;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { MinSyncEmbedderConfig } from "../minsync/types.js";
|
|
2
|
+
import type { RemoteEmbedderSettings } from "./managed.js";
|
|
3
|
+
/**
|
|
4
|
+
* The remote endpoint that `minSync.embedder` describes, if any. The same block configures MinSync itself
|
|
5
|
+
* and every AutoRAG-owned consumer (memory, chat sidecars, gist, discrawl), so one remote setting reaches
|
|
6
|
+
* all of them. `tei:` ids and a `baseUrl` mean a remote server; `openai:` ids default to api.openai.com;
|
|
7
|
+
* `native:` ids, a missing id, and a missing endpoint mean MinSync's local Qwen3.
|
|
8
|
+
*/
|
|
9
|
+
export declare function remoteEmbedderFromMinSync(embedder: MinSyncEmbedderConfig | undefined): RemoteEmbedderSettings | undefined;
|