codesearch-mcp 0.0.18-alpha → 0.1.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +75 -6
- package/dist/app.js +36 -0
- package/dist/src/commands/IndexCommand.js +14 -1
- package/dist/src/commands/McpCommand.js +6 -2
- package/dist/src/config-store.d.ts +49 -11
- package/dist/src/config-store.js +79 -19
- package/dist/src/config.d.ts +32 -8
- package/dist/src/config.js +84 -20
- package/dist/src/constants.d.ts +36 -2
- package/dist/src/constants.js +58 -7
- package/dist/src/container.js +5 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +111 -0
- package/dist/src/db/schema.js +15 -1
- package/dist/src/help.js +5 -4
- package/dist/src/interfaces/IAstParser.d.ts +14 -0
- package/dist/src/interfaces/IChunker.d.ts +24 -0
- package/dist/src/interfaces/IChunker.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +14 -0
- package/dist/src/interfaces/IEmbedder.d.ts +14 -2
- package/dist/src/interfaces/ISearcher.d.ts +1 -0
- package/dist/src/interfaces/index.d.ts +2 -1
- package/dist/src/owner-path.d.ts +10 -0
- package/dist/src/owner-path.js +27 -0
- package/dist/src/services/AstChunker.d.ts +31 -0
- package/dist/src/services/AstChunker.js +114 -0
- package/dist/src/services/AstParserService.d.ts +6 -0
- package/dist/src/services/AstParserService.js +31 -2
- package/dist/src/services/DatabaseService.d.ts +13 -1
- package/dist/src/services/DatabaseService.js +70 -25
- package/dist/src/services/EmbedderFactory.d.ts +9 -0
- package/dist/src/services/EmbedderFactory.js +29 -0
- package/dist/src/services/IndexerService.d.ts +17 -5
- package/dist/src/services/IndexerService.js +101 -38
- package/dist/src/services/InfinityEmbedder.d.ts +79 -0
- package/dist/src/services/InfinityEmbedder.js +199 -0
- package/dist/src/services/OllamaEmbedder.d.ts +16 -5
- package/dist/src/services/OllamaEmbedder.js +84 -36
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
- package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
- package/dist/src/services/SearcherService.d.ts +7 -1
- package/dist/src/services/SearcherService.js +35 -16
- package/dist/src/services/VoyageEmbedder.d.ts +22 -0
- package/dist/src/services/VoyageEmbedder.js +72 -0
- package/dist/src/services/WebServerService.d.ts +16 -11
- package/dist/src/services/WebServerService.js +44 -24
- package/dist/src/services/http-retry.d.ts +19 -0
- package/dist/src/services/http-retry.js +54 -0
- package/dist/src/services/reranker.d.ts +15 -0
- package/dist/src/services/reranker.js +44 -0
- package/dist/src/setup.d.ts +6 -0
- package/dist/src/setup.js +243 -25
- package/dist/src/tools/SearchCodebaseTool.js +1 -1
- package/dist/src/tools/format-helpers.js +2 -1
- package/dist/src/web/dashboard.js +5 -0
- package/docker/docker-compose.infinity-gpu.yml +19 -0
- package/docker/docker-compose.yml +25 -0
- package/package.json +21 -2
- package/dist/src/ast.d.ts +0 -2
- package/dist/src/ast.js +0 -110
- package/dist/src/commands/SetupCommand.d.ts +0 -7
- package/dist/src/commands/SetupCommand.js +0 -14
- package/dist/src/db/data-source.d.ts +0 -2
- package/dist/src/db/data-source.js +0 -13
- package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
- package/dist/src/db/entities/code-chunk.entity.js +0 -54
- package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
- package/dist/src/db/entities/code-dependency.entity.js +0 -54
- package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
- package/dist/src/db/entities/code-symbol.entity.js +0 -59
- package/dist/src/db/init.d.ts +0 -2
- package/dist/src/db/init.js +0 -62
- package/dist/src/env-loader.d.ts +0 -1
- package/dist/src/env-loader.js +0 -41
- package/dist/src/indexer.d.ts +0 -17
- package/dist/src/indexer.js +0 -242
- package/dist/src/mcp.d.ts +0 -1
- package/dist/src/mcp.js +0 -403
- package/dist/src/ollama.d.ts +0 -2
- package/dist/src/ollama.js +0 -6
- package/dist/src/searcher.d.ts +0 -74
- package/dist/src/searcher.js +0 -446
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -9
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
export interface EmbedderDeps {
|
|
5
|
+
fetchImpl?: typeof fetch;
|
|
6
|
+
baseDelayMs?: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Embedder for any server exposing an OpenAI-style `/embeddings` endpoint:
|
|
10
|
+
* OpenAI cloud, Infinity, TEI, vLLM, LM Studio, LocalAI. One request carries all
|
|
11
|
+
* inputs (`{ input: string[], model, dimensions? }`) and returns
|
|
12
|
+
* `{ data: [{ embedding, index }] }`, re-ordered to input order.
|
|
13
|
+
*/
|
|
14
|
+
export declare class OpenAICompatibleEmbedder implements IEmbedder {
|
|
15
|
+
protected config: Config;
|
|
16
|
+
private logger;
|
|
17
|
+
readonly dimensions: number;
|
|
18
|
+
rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
|
|
19
|
+
private readonly url;
|
|
20
|
+
private readonly doFetch;
|
|
21
|
+
private readonly baseDelayMs?;
|
|
22
|
+
constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
|
|
23
|
+
embed(text: string, inputType?: InputType): Promise<number[]>;
|
|
24
|
+
embedBatch(texts: string[], _inputType?: InputType): Promise<number[][]>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.OpenAICompatibleEmbedder = void 0;
|
|
4
|
+
const LoggerService_1 = require("./LoggerService");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const reranker_1 = require("./reranker");
|
|
7
|
+
const http_retry_1 = require("./http-retry");
|
|
8
|
+
const constants_1 = require("../constants");
|
|
9
|
+
/**
|
|
10
|
+
* Embedder for any server exposing an OpenAI-style `/embeddings` endpoint:
|
|
11
|
+
* OpenAI cloud, Infinity, TEI, vLLM, LM Studio, LocalAI. One request carries all
|
|
12
|
+
* inputs (`{ input: string[], model, dimensions? }`) and returns
|
|
13
|
+
* `{ data: [{ embedding, index }] }`, re-ordered to input order.
|
|
14
|
+
*/
|
|
15
|
+
class OpenAICompatibleEmbedder {
|
|
16
|
+
config;
|
|
17
|
+
logger;
|
|
18
|
+
dimensions;
|
|
19
|
+
rerank;
|
|
20
|
+
url;
|
|
21
|
+
doFetch;
|
|
22
|
+
baseDelayMs;
|
|
23
|
+
constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.logger = logger;
|
|
26
|
+
const base = config.embedding.baseURL.replace(/\/+$/, '');
|
|
27
|
+
// Allow a baseURL that already includes the path (…/v1) or not.
|
|
28
|
+
this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
|
|
29
|
+
this.doFetch = deps.fetchImpl ?? fetch;
|
|
30
|
+
this.baseDelayMs = deps.baseDelayMs;
|
|
31
|
+
this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
|
|
32
|
+
this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
|
|
33
|
+
}
|
|
34
|
+
async embed(text, inputType) {
|
|
35
|
+
const [v] = await this.embedBatch([text], inputType);
|
|
36
|
+
return v;
|
|
37
|
+
}
|
|
38
|
+
async embedBatch(texts, _inputType) {
|
|
39
|
+
if (texts.length === 0)
|
|
40
|
+
return [];
|
|
41
|
+
return (0, http_retry_1.withRetry)(async () => {
|
|
42
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
43
|
+
// Resolve the key lazily — an unset `env:` ref throws here (request time),
|
|
44
|
+
// not at construction, so `rag-web`/`rag-mcp` still start.
|
|
45
|
+
const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
|
|
46
|
+
if (apiKey)
|
|
47
|
+
headers['Authorization'] = `Bearer ${apiKey}`;
|
|
48
|
+
const body = { model: this.config.embedding.model, input: texts };
|
|
49
|
+
if (this.config.embedding.dimensions != null)
|
|
50
|
+
body.dimensions = this.config.embedding.dimensions;
|
|
51
|
+
const res = await this.doFetch(this.url, {
|
|
52
|
+
method: 'POST',
|
|
53
|
+
headers,
|
|
54
|
+
signal: AbortSignal.timeout(120_000),
|
|
55
|
+
body: JSON.stringify(body),
|
|
56
|
+
});
|
|
57
|
+
if (!res.ok) {
|
|
58
|
+
const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
|
|
59
|
+
throw new http_retry_1.HttpError(res.status, `embeddings request failed: ${res.status} ${errBody}`);
|
|
60
|
+
}
|
|
61
|
+
const data = await res.json();
|
|
62
|
+
// Guard against a partial/mismatched response — a missing vector would
|
|
63
|
+
// otherwise be inserted as "[undefined]" and fail the whole file, or (with
|
|
64
|
+
// reordering + no index) silently misalign vectors to the wrong chunks.
|
|
65
|
+
if (!Array.isArray(data.data) || data.data.length !== texts.length) {
|
|
66
|
+
throw new Error(`embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
|
|
67
|
+
}
|
|
68
|
+
// Sort by `index` so vectors line up with input order regardless of server ordering.
|
|
69
|
+
return [...data.data]
|
|
70
|
+
.sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
|
|
71
|
+
.map(d => d.embedding);
|
|
72
|
+
}, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'openai-embed' });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.OpenAICompatibleEmbedder = OpenAICompatibleEmbedder;
|
|
@@ -9,6 +9,7 @@ export interface ChunkRow {
|
|
|
9
9
|
filePath: string;
|
|
10
10
|
startLine: number;
|
|
11
11
|
content: string;
|
|
12
|
+
ownerPath?: string | null;
|
|
12
13
|
}
|
|
13
14
|
export interface SearchResult extends ChunkRow {
|
|
14
15
|
score: number;
|
|
@@ -102,6 +103,12 @@ export declare class SearcherService {
|
|
|
102
103
|
query: string;
|
|
103
104
|
translated: boolean;
|
|
104
105
|
}>;
|
|
106
|
+
/**
|
|
107
|
+
* Refuse when the configured embedding model disagrees with the one that built
|
|
108
|
+
* the index — embedding a query into a foreign vector space returns noise. A
|
|
109
|
+
* missing `index_meta` (un-migrated DB / not yet indexed) does not block.
|
|
110
|
+
*/
|
|
111
|
+
assertIndexMatches(): Promise<void>;
|
|
105
112
|
listProjects(): Promise<ProjectRow[]>;
|
|
106
113
|
searchChunks(query: string, limit: number, project: string | null, threshold?: number): Promise<SearchResult[]>;
|
|
107
114
|
fetchNeighbors(results: SearchResult[]): Promise<NeighborResult[]>;
|
|
@@ -125,7 +132,6 @@ export declare class SearcherService {
|
|
|
125
132
|
};
|
|
126
133
|
readFileAbsolute(absPath: string, maxLines: number): Promise<FileRangeResult>;
|
|
127
134
|
readFileRange(filePath: string, startLine: number, endLine: number): Promise<FileRangeResult>;
|
|
128
|
-
private isNonEnglish;
|
|
129
135
|
private walkDir;
|
|
130
136
|
private buildFindSymbolPattern;
|
|
131
137
|
private inferKind;
|
|
@@ -127,16 +127,37 @@ class SearcherService {
|
|
|
127
127
|
}
|
|
128
128
|
return edges;
|
|
129
129
|
}
|
|
130
|
+
// Query translation was removed (legacy). The query is embedded as-is; the
|
|
131
|
+
// method is kept so the tool layer needs no change and `translated` is always
|
|
132
|
+
// false.
|
|
130
133
|
async prepareQuery(raw) {
|
|
131
|
-
const t0 = Date.now();
|
|
132
|
-
if (this.config.search.autoTranslate && this.isNonEnglish(raw) && this.embedder.translate) {
|
|
133
|
-
const query = await this.embedder.translate(raw, this.config.search.translateModel);
|
|
134
|
-
this.logger.debug('SearcherService: prepareQuery', { translated: true, raw: raw.slice(0, 120), query: query.slice(0, 120), ms: Date.now() - t0 });
|
|
135
|
-
return { query, translated: true };
|
|
136
|
-
}
|
|
137
|
-
this.logger.debug('SearcherService: prepareQuery', { translated: false, query: raw.slice(0, 120), ms: Date.now() - t0 });
|
|
138
134
|
return { query: raw, translated: false };
|
|
139
135
|
}
|
|
136
|
+
/**
|
|
137
|
+
* Refuse when the configured embedding model disagrees with the one that built
|
|
138
|
+
* the index — embedding a query into a foreign vector space returns noise. A
|
|
139
|
+
* missing `index_meta` (un-migrated DB / not yet indexed) does not block.
|
|
140
|
+
*/
|
|
141
|
+
async assertIndexMatches() {
|
|
142
|
+
const meta = await this.db.readIndexMeta();
|
|
143
|
+
if (!meta)
|
|
144
|
+
return;
|
|
145
|
+
const { provider, model } = this.config.embedding;
|
|
146
|
+
if (meta.provider !== provider || meta.model !== model) {
|
|
147
|
+
throw new Error(`Index was built with ${meta.provider}/${meta.model} but the config selects ${provider}/${model}. ` +
|
|
148
|
+
`Run \`rag-index --reindex\` to rebuild the index for the new model.`);
|
|
149
|
+
}
|
|
150
|
+
// Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
|
|
151
|
+
// override) would embed the query at a width the column can't hold — pgvector
|
|
152
|
+
// would throw an opaque "different vector dimensions" error mid-query. Catch
|
|
153
|
+
// it here with the friendly hint. `embedder.dimensions` is 0 when the width is
|
|
154
|
+
// only knowable by probing, in which case we skip (can't compare cheaply).
|
|
155
|
+
const dim = this.embedder.dimensions;
|
|
156
|
+
if (dim > 0 && dim !== meta.dimensions) {
|
|
157
|
+
throw new Error(`Index was built at ${meta.dimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
|
|
158
|
+
`Run \`rag-index --reindex\` to rebuild the index at the new dimension.`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
140
161
|
async listProjects() {
|
|
141
162
|
const rows = await this.db.query(`SELECT "serviceName", COUNT(*)::int AS chunks
|
|
142
163
|
FROM code_chunk
|
|
@@ -150,21 +171,22 @@ class SearcherService {
|
|
|
150
171
|
}
|
|
151
172
|
async searchChunks(query, limit, project, threshold) {
|
|
152
173
|
const t0 = Date.now();
|
|
174
|
+
await this.assertIndexMatches();
|
|
153
175
|
const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
|
|
154
176
|
const fetchLimit = this.embedder.rerank
|
|
155
|
-
? Math.min(limit * this.config.
|
|
177
|
+
? Math.min(limit * this.config.reranker.overfetch, 200)
|
|
156
178
|
: limit;
|
|
157
179
|
this.logger.debug('SearcherService: searchChunks', {
|
|
158
180
|
query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
|
|
159
181
|
});
|
|
160
|
-
const embedding = await this.embedder.embed(query);
|
|
182
|
+
const embedding = await this.embedder.embed(query, 'query');
|
|
161
183
|
const params = [`[${embedding}]`, effectiveThreshold, fetchLimit];
|
|
162
184
|
let projectFilter = '';
|
|
163
185
|
if (project) {
|
|
164
186
|
params.push(project);
|
|
165
187
|
projectFilter = `AND "serviceName" = $${params.length}`;
|
|
166
188
|
}
|
|
167
|
-
const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content,
|
|
189
|
+
const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
|
|
168
190
|
ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
|
|
169
191
|
FROM code_chunk
|
|
170
192
|
WHERE embedding <=> $1 < $2
|
|
@@ -182,7 +204,9 @@ class SearcherService {
|
|
|
182
204
|
try {
|
|
183
205
|
const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
|
|
184
206
|
this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
|
|
185
|
-
|
|
207
|
+
// Cap at `limit`: some rerank servers ignore `top_n` and echo back every
|
|
208
|
+
// candidate, so slicing here keeps the result count within the request.
|
|
209
|
+
return indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
|
|
186
210
|
}
|
|
187
211
|
catch (err) {
|
|
188
212
|
this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
|
|
@@ -602,11 +626,6 @@ class SearcherService {
|
|
|
602
626
|
this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
|
|
603
627
|
return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
|
|
604
628
|
}
|
|
605
|
-
isNonEnglish(text) {
|
|
606
|
-
// Use Unicode script detection: check for CJK, Cyrillic, Arabic, Hebrew, Thai, etc.
|
|
607
|
-
// Excludes common non-ASCII punctuation (smart quotes, em-dashes, accented Latin chars)
|
|
608
|
-
return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Cyrillic}\p{Script=Arabic}\p{Script=Hebrew}\p{Script=Thai}\p{Script=Devanagari}]/u.test(text);
|
|
609
|
-
}
|
|
610
629
|
walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
|
|
611
630
|
let entries;
|
|
612
631
|
try {
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
5
|
+
/**
|
|
6
|
+
* Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
|
|
7
|
+
* of strings, `input_type` (`query` | `document`) is required for best quality,
|
|
8
|
+
* and the optional `output_dimension` requests Matryoshka truncation. Response
|
|
9
|
+
* is `{ data: [{ embedding, index }] }`.
|
|
10
|
+
*/
|
|
11
|
+
export declare class VoyageEmbedder implements IEmbedder {
|
|
12
|
+
private config;
|
|
13
|
+
private logger;
|
|
14
|
+
readonly dimensions: number;
|
|
15
|
+
rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
|
|
16
|
+
private readonly url;
|
|
17
|
+
private readonly doFetch;
|
|
18
|
+
private readonly baseDelayMs?;
|
|
19
|
+
constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
|
|
20
|
+
embed(text: string, inputType?: InputType): Promise<number[]>;
|
|
21
|
+
embedBatch(texts: string[], inputType?: InputType): Promise<number[][]>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.VoyageEmbedder = void 0;
|
|
4
|
+
const LoggerService_1 = require("./LoggerService");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const reranker_1 = require("./reranker");
|
|
7
|
+
const http_retry_1 = require("./http-retry");
|
|
8
|
+
const constants_1 = require("../constants");
|
|
9
|
+
/**
|
|
10
|
+
* Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
|
|
11
|
+
* of strings, `input_type` (`query` | `document`) is required for best quality,
|
|
12
|
+
* and the optional `output_dimension` requests Matryoshka truncation. Response
|
|
13
|
+
* is `{ data: [{ embedding, index }] }`.
|
|
14
|
+
*/
|
|
15
|
+
class VoyageEmbedder {
|
|
16
|
+
config;
|
|
17
|
+
logger;
|
|
18
|
+
dimensions;
|
|
19
|
+
rerank;
|
|
20
|
+
url;
|
|
21
|
+
doFetch;
|
|
22
|
+
baseDelayMs;
|
|
23
|
+
constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.logger = logger;
|
|
26
|
+
const base = config.embedding.baseURL.replace(/\/+$/, '');
|
|
27
|
+
this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
|
|
28
|
+
this.doFetch = deps.fetchImpl ?? fetch;
|
|
29
|
+
this.baseDelayMs = deps.baseDelayMs;
|
|
30
|
+
this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
|
|
31
|
+
this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
|
|
32
|
+
}
|
|
33
|
+
async embed(text, inputType) {
|
|
34
|
+
const [v] = await this.embedBatch([text], inputType);
|
|
35
|
+
return v;
|
|
36
|
+
}
|
|
37
|
+
async embedBatch(texts, inputType = 'document') {
|
|
38
|
+
if (texts.length === 0)
|
|
39
|
+
return [];
|
|
40
|
+
return (0, http_retry_1.withRetry)(async () => {
|
|
41
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
42
|
+
const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
|
|
43
|
+
if (apiKey)
|
|
44
|
+
headers['Authorization'] = `Bearer ${apiKey}`;
|
|
45
|
+
const body = {
|
|
46
|
+
model: this.config.embedding.model,
|
|
47
|
+
input: texts,
|
|
48
|
+
input_type: inputType,
|
|
49
|
+
};
|
|
50
|
+
if (this.config.embedding.dimensions != null)
|
|
51
|
+
body.output_dimension = this.config.embedding.dimensions;
|
|
52
|
+
const res = await this.doFetch(this.url, {
|
|
53
|
+
method: 'POST',
|
|
54
|
+
headers,
|
|
55
|
+
signal: AbortSignal.timeout(120_000),
|
|
56
|
+
body: JSON.stringify(body),
|
|
57
|
+
});
|
|
58
|
+
if (!res.ok) {
|
|
59
|
+
const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
|
|
60
|
+
throw new http_retry_1.HttpError(res.status, `Voyage embeddings request failed: ${res.status} ${errBody}`);
|
|
61
|
+
}
|
|
62
|
+
const data = await res.json();
|
|
63
|
+
if (!Array.isArray(data.data) || data.data.length !== texts.length) {
|
|
64
|
+
throw new Error(`Voyage embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
|
|
65
|
+
}
|
|
66
|
+
return [...data.data]
|
|
67
|
+
.sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
|
|
68
|
+
.map(d => d.embedding);
|
|
69
|
+
}, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'voyage-embed' });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
exports.VoyageEmbedder = VoyageEmbedder;
|
|
@@ -13,19 +13,22 @@ export interface FieldOption {
|
|
|
13
13
|
values: string[];
|
|
14
14
|
strict: boolean;
|
|
15
15
|
}
|
|
16
|
-
/** Placeholder shown to the browser instead of
|
|
16
|
+
/** Placeholder shown to the browser instead of a real secret. */
|
|
17
17
|
export declare const PASSWORD_MASK = "********";
|
|
18
18
|
/**
|
|
19
|
-
* Merge a config submitted from the browser with the stored one.
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* placeholder.
|
|
19
|
+
* Merge a config submitted from the browser with the stored one. Any secret
|
|
20
|
+
* field (DB password, embedding/reranker API keys) still holding the mask means
|
|
21
|
+
* the user did not change it — restore the stored value rather than overwrite it
|
|
22
|
+
* with the placeholder. `env:` references are not masked and round-trip as-is.
|
|
23
|
+
* Pure — no I/O. Exported for unit testing.
|
|
23
24
|
*/
|
|
24
25
|
export declare function mergeConfigForSave(incoming: JsonConfig, existing: JsonConfig | null): JsonConfig;
|
|
25
26
|
/**
|
|
26
|
-
* Return a copy of the config with
|
|
27
|
-
* real
|
|
28
|
-
*
|
|
27
|
+
* Return a copy of the config with raw secrets (DB password, raw API keys)
|
|
28
|
+
* replaced by the mask, so real secrets never reach the browser. `env:`
|
|
29
|
+
* references are left verbatim — they are not secret and the dashboard shows
|
|
30
|
+
* them as "set via <VAR>". The masked value round-trips through
|
|
31
|
+
* `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
|
|
29
32
|
*/
|
|
30
33
|
export declare function maskConfig(cfg: JsonConfig): JsonConfig;
|
|
31
34
|
/**
|
|
@@ -49,9 +52,11 @@ export declare class WebServer {
|
|
|
49
52
|
private maskedConfig;
|
|
50
53
|
private saveConfig;
|
|
51
54
|
/**
|
|
52
|
-
* Choice lists for model fields.
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
+
* Choice lists for model fields. Both are non-strict comboboxes (free text
|
|
56
|
+
* allowed): the embedding model suggests the known registry models plus any
|
|
57
|
+
* locally-installed models; the reranker model suggests installed models.
|
|
58
|
+
* Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
|
|
59
|
+
* still work — their dimension is probed at index time.
|
|
55
60
|
*/
|
|
56
61
|
private fieldOptions;
|
|
57
62
|
private installedModels;
|
|
@@ -42,35 +42,53 @@ const config_store_1 = require("../config-store");
|
|
|
42
42
|
const constants_1 = require("../constants");
|
|
43
43
|
const graph_1 = require("../graph");
|
|
44
44
|
const dashboard_1 = require("../web/dashboard");
|
|
45
|
-
/** Placeholder shown to the browser instead of
|
|
45
|
+
/** Placeholder shown to the browser instead of a real secret. */
|
|
46
46
|
exports.PASSWORD_MASK = '********';
|
|
47
47
|
const MAX_BODY = 1_000_000; // 1 MB — config payloads are tiny
|
|
48
|
+
/** A secret worth masking: a non-empty string that is not an `env:` reference. */
|
|
49
|
+
function isRawSecret(v) {
|
|
50
|
+
return typeof v === 'string' && v !== '' && !v.startsWith('env:');
|
|
51
|
+
}
|
|
48
52
|
/**
|
|
49
|
-
* Merge a config submitted from the browser with the stored one.
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* placeholder.
|
|
53
|
+
* Merge a config submitted from the browser with the stored one. Any secret
|
|
54
|
+
* field (DB password, embedding/reranker API keys) still holding the mask means
|
|
55
|
+
* the user did not change it — restore the stored value rather than overwrite it
|
|
56
|
+
* with the placeholder. `env:` references are not masked and round-trip as-is.
|
|
57
|
+
* Pure — no I/O. Exported for unit testing.
|
|
53
58
|
*/
|
|
54
59
|
function mergeConfigForSave(incoming, existing) {
|
|
55
60
|
const merged = JSON.parse(JSON.stringify(incoming ?? {}));
|
|
56
|
-
|
|
57
|
-
const
|
|
58
|
-
if (
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
const restore = (grp, field) => {
|
|
62
|
+
const g = merged[grp];
|
|
63
|
+
if (g && g[field] === exports.PASSWORD_MASK) {
|
|
64
|
+
const prev = existing?.[grp]?.[field];
|
|
65
|
+
if (prev !== undefined)
|
|
66
|
+
g[field] = prev;
|
|
67
|
+
else
|
|
68
|
+
delete g[field];
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
restore('db', 'password');
|
|
72
|
+
restore('embedding', 'apiKey');
|
|
73
|
+
restore('reranker', 'apiKey');
|
|
63
74
|
return merged;
|
|
64
75
|
}
|
|
65
76
|
/**
|
|
66
|
-
* Return a copy of the config with
|
|
67
|
-
* real
|
|
68
|
-
*
|
|
77
|
+
* Return a copy of the config with raw secrets (DB password, raw API keys)
|
|
78
|
+
* replaced by the mask, so real secrets never reach the browser. `env:`
|
|
79
|
+
* references are left verbatim — they are not secret and the dashboard shows
|
|
80
|
+
* them as "set via <VAR>". The masked value round-trips through
|
|
81
|
+
* `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
|
|
69
82
|
*/
|
|
70
83
|
function maskConfig(cfg) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
84
|
+
const clone = JSON.parse(JSON.stringify(cfg ?? {}));
|
|
85
|
+
if (clone.db?.password)
|
|
86
|
+
clone.db.password = exports.PASSWORD_MASK;
|
|
87
|
+
if (isRawSecret(clone.embedding?.apiKey))
|
|
88
|
+
clone.embedding.apiKey = exports.PASSWORD_MASK;
|
|
89
|
+
if (isRawSecret(clone.reranker?.apiKey))
|
|
90
|
+
clone.reranker.apiKey = exports.PASSWORD_MASK;
|
|
91
|
+
return clone;
|
|
74
92
|
}
|
|
75
93
|
/**
|
|
76
94
|
* Local HTTP dashboard. Serves a schema-driven config editor, a project
|
|
@@ -171,15 +189,17 @@ class WebServer {
|
|
|
171
189
|
this.json(res, 200, { ok: true });
|
|
172
190
|
}
|
|
173
191
|
/**
|
|
174
|
-
* Choice lists for model fields.
|
|
175
|
-
*
|
|
176
|
-
*
|
|
192
|
+
* Choice lists for model fields. Both are non-strict comboboxes (free text
|
|
193
|
+
* allowed): the embedding model suggests the known registry models plus any
|
|
194
|
+
* locally-installed models; the reranker model suggests installed models.
|
|
195
|
+
* Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
|
|
196
|
+
* still work — their dimension is probed at index time.
|
|
177
197
|
*/
|
|
178
198
|
async fieldOptions() {
|
|
179
199
|
const installed = await this.installedModels();
|
|
200
|
+
const registryModels = Object.keys(constants_1.MODEL_REGISTRY);
|
|
180
201
|
return {
|
|
181
|
-
'
|
|
182
|
-
'search.translateModel': { values: installed, strict: false },
|
|
202
|
+
'embedding.model': { values: [...new Set([...registryModels, ...installed])], strict: false },
|
|
183
203
|
'reranker.model': { values: installed, strict: false },
|
|
184
204
|
};
|
|
185
205
|
}
|
|
@@ -188,7 +208,7 @@ class WebServer {
|
|
|
188
208
|
return (await this.embedder.listModels?.()) ?? [];
|
|
189
209
|
}
|
|
190
210
|
catch (err) {
|
|
191
|
-
this.logger.warn('WebServer: could not list
|
|
211
|
+
this.logger.warn('WebServer: could not list models', { reason: err instanceof Error ? err.message : String(err) });
|
|
192
212
|
return [];
|
|
193
213
|
}
|
|
194
214
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
2
|
+
export declare class HttpError extends Error {
|
|
3
|
+
status: number;
|
|
4
|
+
constructor(status: number, message: string);
|
|
5
|
+
}
|
|
6
|
+
export interface RetryOpts {
|
|
7
|
+
/** Total attempts including the first (default 4). */
|
|
8
|
+
attempts?: number;
|
|
9
|
+
/** Base backoff in ms; grows exponentially (default 500). Set 0 in tests. */
|
|
10
|
+
baseDelayMs?: number;
|
|
11
|
+
logger?: ILogger;
|
|
12
|
+
label?: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Run `fn` with bounded exponential backoff on HTTP 429/5xx and network errors.
|
|
16
|
+
* Re-throws the last error once attempts are exhausted or the error is not
|
|
17
|
+
* transient.
|
|
18
|
+
*/
|
|
19
|
+
export declare function withRetry<T>(fn: () => Promise<T>, opts?: RetryOpts): Promise<T>;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HttpError = void 0;
|
|
4
|
+
exports.withRetry = withRetry;
|
|
5
|
+
class HttpError extends Error {
|
|
6
|
+
status;
|
|
7
|
+
constructor(status, message) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.status = status;
|
|
10
|
+
this.name = 'HttpError';
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
exports.HttpError = HttpError;
|
|
14
|
+
function sleep(ms) {
|
|
15
|
+
return ms > 0 ? new Promise(r => setTimeout(r, ms)) : Promise.resolve();
|
|
16
|
+
}
|
|
17
|
+
/** True for transient failures worth retrying (rate limit / server error / network / timeout). */
|
|
18
|
+
function isRetryable(err) {
|
|
19
|
+
if (err instanceof HttpError)
|
|
20
|
+
return err.status === 429 || (err.status >= 500 && err.status <= 599);
|
|
21
|
+
// A request timeout (AbortSignal.timeout) rejects with a DOMException named
|
|
22
|
+
// 'TimeoutError' — the most common transient cloud failure, so retry it.
|
|
23
|
+
if (err != null && typeof err === 'object' && err.name === 'TimeoutError')
|
|
24
|
+
return true;
|
|
25
|
+
// Network-level errors (fetch throws) are transient too.
|
|
26
|
+
return err instanceof TypeError;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Run `fn` with bounded exponential backoff on HTTP 429/5xx and network errors.
|
|
30
|
+
* Re-throws the last error once attempts are exhausted or the error is not
|
|
31
|
+
* transient.
|
|
32
|
+
*/
|
|
33
|
+
async function withRetry(fn, opts = {}) {
|
|
34
|
+
const attempts = opts.attempts ?? 4;
|
|
35
|
+
const base = opts.baseDelayMs ?? 500;
|
|
36
|
+
let lastErr;
|
|
37
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
38
|
+
try {
|
|
39
|
+
return await fn();
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
lastErr = err;
|
|
43
|
+
if (attempt >= attempts || !isRetryable(err))
|
|
44
|
+
break;
|
|
45
|
+
const delay = base * Math.pow(2, attempt - 1);
|
|
46
|
+
opts.logger?.warn('http-retry: transient failure, retrying', {
|
|
47
|
+
label: opts.label, attempt, attempts, delayMs: delay,
|
|
48
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
49
|
+
});
|
|
50
|
+
await sleep(delay);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
throw lastErr;
|
|
54
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
3
|
+
/**
|
|
4
|
+
* Build a rerank function from `config.reranker`, or return `undefined` when
|
|
5
|
+
* reranking is disabled / unconfigured. Targets an OpenAI-compatible `/rerank`
|
|
6
|
+
* endpoint returning `{ results: [{ index }] }` (Infinity, TEI, vLLM,
|
|
7
|
+
* Jina/Cohere-shape). Ollama has NO rerank API, so it is not a valid backend
|
|
8
|
+
* here — the embedding provider is independent of the rerank server
|
|
9
|
+
* (`reranker.baseURL`), so Ollama embeddings + an Infinity reranker is fine.
|
|
10
|
+
* Returns original indices into `documents`, best-first.
|
|
11
|
+
*/
|
|
12
|
+
export declare function makeReranker(config: Config, logger: ILogger, deps?: {
|
|
13
|
+
fetchImpl?: typeof fetch;
|
|
14
|
+
baseDelayMs?: number;
|
|
15
|
+
}): ((query: string, documents: string[], topN: number) => Promise<number[]>) | undefined;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.makeReranker = makeReranker;
|
|
4
|
+
const config_1 = require("../config");
|
|
5
|
+
const http_retry_1 = require("./http-retry");
|
|
6
|
+
/**
|
|
7
|
+
* Build a rerank function from `config.reranker`, or return `undefined` when
|
|
8
|
+
* reranking is disabled / unconfigured. Targets an OpenAI-compatible `/rerank`
|
|
9
|
+
* endpoint returning `{ results: [{ index }] }` (Infinity, TEI, vLLM,
|
|
10
|
+
* Jina/Cohere-shape). Ollama has NO rerank API, so it is not a valid backend
|
|
11
|
+
* here — the embedding provider is independent of the rerank server
|
|
12
|
+
* (`reranker.baseURL`), so Ollama embeddings + an Infinity reranker is fine.
|
|
13
|
+
* Returns original indices into `documents`, best-first.
|
|
14
|
+
*/
|
|
15
|
+
function makeReranker(config, logger, deps = {}) {
|
|
16
|
+
const r = config.reranker;
|
|
17
|
+
if (!r.enabled)
|
|
18
|
+
return undefined;
|
|
19
|
+
if (!r.model) {
|
|
20
|
+
logger.warn('reranker: enabled=true but reranker.model is not set — reranking disabled');
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
const doFetch = deps.fetchImpl ?? fetch;
|
|
24
|
+
return async (query, documents, topN) => {
|
|
25
|
+
return (0, http_retry_1.withRetry)(async () => {
|
|
26
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
27
|
+
const apiKey = (0, config_1.resolveEnvRef)(r.apiKey);
|
|
28
|
+
if (apiKey)
|
|
29
|
+
headers['Authorization'] = `Bearer ${apiKey}`;
|
|
30
|
+
const res = await doFetch(r.url, {
|
|
31
|
+
method: 'POST',
|
|
32
|
+
headers,
|
|
33
|
+
signal: AbortSignal.timeout(30_000),
|
|
34
|
+
body: JSON.stringify({ model: r.model, query, documents, top_n: topN, truncate: true }),
|
|
35
|
+
});
|
|
36
|
+
if (!res.ok) {
|
|
37
|
+
const body = (await res.text().catch(() => res.statusText)).slice(0, 200);
|
|
38
|
+
throw new http_retry_1.HttpError(res.status, `rerank failed: ${res.status} ${body}`);
|
|
39
|
+
}
|
|
40
|
+
const data = await res.json();
|
|
41
|
+
return data.results.map(x => x.index);
|
|
42
|
+
}, { logger, baseDelayMs: deps.baseDelayMs, label: 'rerank' });
|
|
43
|
+
};
|
|
44
|
+
}
|
package/dist/src/setup.d.ts
CHANGED
|
@@ -1,2 +1,8 @@
|
|
|
1
1
|
import type { LoggerService } from './services/LoggerService';
|
|
2
|
+
/**
|
|
3
|
+
* Default answer for the "start Docker containers?" prompt: no only when both
|
|
4
|
+
* the database and the embedding backend are already reachable. Pure — exported
|
|
5
|
+
* for unit testing.
|
|
6
|
+
*/
|
|
7
|
+
export declare function dockerPromptDefault(dbReachable: boolean, embReachable: boolean): 'y' | 'n';
|
|
2
8
|
export declare function runSetup(logger?: LoggerService): Promise<void>;
|