codesearch-mcp 0.0.18-alpha → 0.2.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -1
- package/README.md +194 -23
- package/dist/app.js +38 -2
- package/dist/src/commands/IndexCommand.js +55 -1
- package/dist/src/commands/McpCommand.js +6 -2
- package/dist/src/config-error.d.ts +7 -0
- package/dist/src/config-error.js +14 -0
- package/dist/src/config-store.d.ts +73 -17
- package/dist/src/config-store.js +142 -27
- package/dist/src/config.d.ts +36 -11
- package/dist/src/config.js +89 -27
- package/dist/src/constants.d.ts +42 -2
- package/dist/src/constants.js +61 -7
- package/dist/src/container.js +5 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +111 -0
- package/dist/src/db/schema.js +15 -1
- package/dist/src/help.js +6 -5
- package/dist/src/interfaces/IAstParser.d.ts +14 -0
- package/dist/src/interfaces/IChunker.d.ts +24 -0
- package/dist/src/interfaces/IChunker.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +31 -0
- package/dist/src/interfaces/IEmbedder.d.ts +14 -2
- package/dist/src/interfaces/ILogger.d.ts +7 -0
- package/dist/src/interfaces/ISearcher.d.ts +1 -0
- package/dist/src/interfaces/index.d.ts +2 -1
- package/dist/src/owner-path.d.ts +10 -0
- package/dist/src/owner-path.js +27 -0
- package/dist/src/services/AstChunker.d.ts +42 -0
- package/dist/src/services/AstChunker.js +158 -0
- package/dist/src/services/AstParserService.d.ts +6 -0
- package/dist/src/services/AstParserService.js +31 -2
- package/dist/src/services/DatabaseService.d.ts +17 -1
- package/dist/src/services/DatabaseService.js +116 -25
- package/dist/src/services/EmbedderFactory.d.ts +9 -0
- package/dist/src/services/EmbedderFactory.js +26 -0
- package/dist/src/services/IndexerService.d.ts +21 -5
- package/dist/src/services/IndexerService.js +127 -44
- package/dist/src/services/InfinityEmbedder.d.ts +38 -0
- package/dist/src/services/InfinityEmbedder.js +88 -0
- package/dist/src/services/LoggerService.d.ts +3 -1
- package/dist/src/services/LoggerService.js +17 -3
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
- package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
- package/dist/src/services/SearcherService.d.ts +21 -1
- package/dist/src/services/SearcherService.js +129 -28
- package/dist/src/services/VoyageEmbedder.d.ts +22 -0
- package/dist/src/services/VoyageEmbedder.js +72 -0
- package/dist/src/services/WebServerService.d.ts +57 -14
- package/dist/src/services/WebServerService.js +134 -26
- package/dist/src/services/http-retry.d.ts +19 -0
- package/dist/src/services/http-retry.js +54 -0
- package/dist/src/services/infinity-provisioning.d.ts +115 -0
- package/dist/src/services/infinity-provisioning.js +516 -0
- package/dist/src/services/legacy-ollama-index.d.ts +24 -0
- package/dist/src/services/legacy-ollama-index.js +61 -0
- package/dist/src/services/reranker.d.ts +15 -0
- package/dist/src/services/reranker.js +44 -0
- package/dist/src/setup.d.ts +43 -0
- package/dist/src/setup.js +227 -55
- package/dist/src/tools/SearchCodebaseTool.js +1 -1
- package/dist/src/tools/format-helpers.js +2 -1
- package/dist/src/web/dashboard.js +5 -0
- package/docker/docker-compose.infinity-gpu.yml +19 -0
- package/docker/docker-compose.yml +20 -5
- package/docs/web-dashboard.md +36 -17
- package/package.json +18 -3
- package/dist/src/ast.d.ts +0 -2
- package/dist/src/ast.js +0 -110
- package/dist/src/commands/SetupCommand.d.ts +0 -7
- package/dist/src/commands/SetupCommand.js +0 -14
- package/dist/src/db/data-source.d.ts +0 -2
- package/dist/src/db/data-source.js +0 -13
- package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
- package/dist/src/db/entities/code-chunk.entity.js +0 -54
- package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
- package/dist/src/db/entities/code-dependency.entity.js +0 -54
- package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
- package/dist/src/db/entities/code-symbol.entity.js +0 -59
- package/dist/src/db/init.d.ts +0 -2
- package/dist/src/db/init.js +0 -62
- package/dist/src/env-loader.d.ts +0 -1
- package/dist/src/env-loader.js +0 -41
- package/dist/src/indexer.d.ts +0 -17
- package/dist/src/indexer.js +0 -242
- package/dist/src/mcp.d.ts +0 -1
- package/dist/src/mcp.js +0 -403
- package/dist/src/ollama.d.ts +0 -2
- package/dist/src/ollama.js +0 -6
- package/dist/src/searcher.d.ts +0 -74
- package/dist/src/searcher.js +0 -446
- package/dist/src/services/OllamaEmbedder.d.ts +0 -14
- package/dist/src/services/OllamaEmbedder.js +0 -63
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -9
|
@@ -40,11 +40,13 @@ const readline = __importStar(require("readline"));
|
|
|
40
40
|
const child_process_1 = require("child_process");
|
|
41
41
|
const file_utils_1 = require("../file-utils");
|
|
42
42
|
const MonorepoAwareDiscoveryService_1 = require("./MonorepoAwareDiscoveryService");
|
|
43
|
+
const legacy_ollama_index_1 = require("./legacy-ollama-index");
|
|
43
44
|
const SILENT_LOGGER = {
|
|
44
45
|
debug() { },
|
|
45
46
|
info() { },
|
|
46
47
|
warn() { },
|
|
47
48
|
error() { },
|
|
49
|
+
metric() { },
|
|
48
50
|
};
|
|
49
51
|
const EMPTY_META = { description: null, stack: null, notes: null, relations: null };
|
|
50
52
|
const GREP_MAX_RESULTS = 200;
|
|
@@ -127,16 +129,53 @@ class SearcherService {
|
|
|
127
129
|
}
|
|
128
130
|
return edges;
|
|
129
131
|
}
|
|
132
|
+
// Query translation was removed (legacy). The query is embedded as-is; the
|
|
133
|
+
// method is kept so the tool layer needs no change and `translated` is always
|
|
134
|
+
// false.
|
|
130
135
|
async prepareQuery(raw) {
|
|
131
|
-
const t0 = Date.now();
|
|
132
|
-
if (this.config.search.autoTranslate && this.isNonEnglish(raw) && this.embedder.translate) {
|
|
133
|
-
const query = await this.embedder.translate(raw, this.config.search.translateModel);
|
|
134
|
-
this.logger.debug('SearcherService: prepareQuery', { translated: true, raw: raw.slice(0, 120), query: query.slice(0, 120), ms: Date.now() - t0 });
|
|
135
|
-
return { query, translated: true };
|
|
136
|
-
}
|
|
137
|
-
this.logger.debug('SearcherService: prepareQuery', { translated: false, query: raw.slice(0, 120), ms: Date.now() - t0 });
|
|
138
136
|
return { query: raw, translated: false };
|
|
139
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* Refuse when the configured embedding model disagrees with the one that built
|
|
140
|
+
* the index — embedding a query into a foreign vector space returns noise. A
|
|
141
|
+
* missing `index_meta` row with an empty (or altogether absent) `code_chunk`
|
|
142
|
+
* does not block — but a *populated* `code_chunk` with no `index_meta` row is a
|
|
143
|
+
* pre-`index_meta` install, treated as the legacy `ollama`/`bge-m3` index (same
|
|
144
|
+
* interpretation as the `rag-index` legacy-Ollama guard), read-only — this check
|
|
145
|
+
* never writes `index_meta`.
|
|
146
|
+
*/
|
|
147
|
+
async assertIndexMatches() {
|
|
148
|
+
const meta = await this.db.readIndexMeta();
|
|
149
|
+
if (meta) {
|
|
150
|
+
this.compareIndexed(meta.provider, meta.model, meta.dimensions);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
// No row — a missing code_chunk table counts as empty; any other DB error
|
|
154
|
+
// while counting propagates rather than being read as "not indexed".
|
|
155
|
+
if (await (0, legacy_ollama_index_1.hasIndexedChunks)(this.db)) {
|
|
156
|
+
this.compareIndexed(legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.provider, legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.model, null);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
compareIndexed(indexedProvider, indexedModel, indexedDimensions) {
|
|
160
|
+
const { provider, model } = this.config.embedding;
|
|
161
|
+
if (indexedProvider !== provider || indexedModel !== model) {
|
|
162
|
+
throw new Error(`Index was built with ${indexedProvider}/${indexedModel} but the config selects ${provider}/${model}. ` +
|
|
163
|
+
`Run \`rag-index --reindex\` to rebuild the index for the new model.`);
|
|
164
|
+
}
|
|
165
|
+
// Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
|
|
166
|
+
// override) would embed the query at a width the column can't hold — pgvector
|
|
167
|
+
// would throw an opaque "different vector dimensions" error mid-query. Catch
|
|
168
|
+
// it here with the friendly hint. `embedder.dimensions` is 0 when the width is
|
|
169
|
+
// only knowable by probing, in which case we skip (can't compare cheaply).
|
|
170
|
+
// `indexedDimensions` is null for the legacy-inferred identity, whose provider
|
|
171
|
+
// can never match a configured provider (ollama is no longer selectable), so
|
|
172
|
+
// this branch is unreached in that case.
|
|
173
|
+
const dim = this.embedder.dimensions;
|
|
174
|
+
if (indexedDimensions != null && dim > 0 && dim !== indexedDimensions) {
|
|
175
|
+
throw new Error(`Index was built at ${indexedDimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
|
|
176
|
+
`Run \`rag-index --reindex\` to rebuild the index at the new dimension.`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
140
179
|
async listProjects() {
|
|
141
180
|
const rows = await this.db.query(`SELECT "serviceName", COUNT(*)::int AS chunks
|
|
142
181
|
FROM code_chunk
|
|
@@ -150,47 +189,114 @@ class SearcherService {
|
|
|
150
189
|
}
|
|
151
190
|
async searchChunks(query, limit, project, threshold) {
|
|
152
191
|
const t0 = Date.now();
|
|
192
|
+
await this.assertIndexMatches();
|
|
153
193
|
const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
|
|
154
|
-
const
|
|
155
|
-
|
|
194
|
+
const reranking = !!this.embedder.rerank;
|
|
195
|
+
// Candidate floor: decouple the pool size from `limit` so a small `limit`
|
|
196
|
+
// (e.g. the default 5) still gives the reranker a large-enough menu. Without
|
|
197
|
+
// a reranker the extra candidates are never reordered, so keep `limit`.
|
|
198
|
+
const fetchLimit = reranking
|
|
199
|
+
? Math.min(Math.max(limit * this.config.reranker.overfetch, this.config.reranker.candidateFloor), 200)
|
|
156
200
|
: limit;
|
|
201
|
+
// With metrics on, fetch one extra row to detect a saturated pool (more
|
|
202
|
+
// under-threshold candidates existed than the window took). The extra row is
|
|
203
|
+
// dropped before reranking so it never affects results.
|
|
204
|
+
const metricsOn = this.config.logging.searchMetrics;
|
|
205
|
+
const dbLimit = metricsOn ? fetchLimit + 1 : fetchLimit;
|
|
157
206
|
this.logger.debug('SearcherService: searchChunks', {
|
|
158
207
|
query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
|
|
159
208
|
});
|
|
160
|
-
const
|
|
161
|
-
const
|
|
209
|
+
const tEmbed0 = Date.now();
|
|
210
|
+
const embedding = await this.embedder.embed(query, 'query');
|
|
211
|
+
const embedMs = Date.now() - tEmbed0;
|
|
212
|
+
const params = [`[${embedding}]`, effectiveThreshold, dbLimit];
|
|
162
213
|
let projectFilter = '';
|
|
163
214
|
if (project) {
|
|
164
215
|
params.push(project);
|
|
165
216
|
projectFilter = `AND "serviceName" = $${params.length}`;
|
|
166
217
|
}
|
|
167
|
-
const
|
|
218
|
+
const tDb0 = Date.now();
|
|
219
|
+
const fetched = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
|
|
168
220
|
ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
|
|
169
221
|
FROM code_chunk
|
|
170
222
|
WHERE embedding <=> $1 < $2
|
|
171
223
|
${projectFilter}
|
|
172
224
|
ORDER BY embedding <=> $1
|
|
173
225
|
LIMIT $3`, params);
|
|
226
|
+
const dbMs = Date.now() - tDb0;
|
|
227
|
+
// Saturated iff the extra probe row came back — more candidates were under
|
|
228
|
+
// the threshold than `fetchLimit`. Trim the probe row before reranking.
|
|
229
|
+
const poolSaturated = metricsOn && fetched.length > fetchLimit;
|
|
230
|
+
const candidates = fetched.slice(0, fetchLimit);
|
|
174
231
|
this.logger.debug('SearcherService: searchChunks candidates', {
|
|
175
232
|
count: candidates.length,
|
|
176
233
|
scores: candidates.map(r => r.score),
|
|
177
234
|
files: candidates.map(r => r.filePath),
|
|
178
235
|
ms: Date.now() - t0,
|
|
179
236
|
});
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
|
|
185
|
-
return indices.filter(i => i >= 0 && i < candidates.length).map(i => candidates[i]);
|
|
237
|
+
let results;
|
|
238
|
+
let rerankMs = 0;
|
|
239
|
+
if (!reranking || candidates.length === 0) {
|
|
240
|
+
results = candidates.slice(0, limit);
|
|
186
241
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
242
|
+
else {
|
|
243
|
+
const tRerank0 = Date.now();
|
|
244
|
+
try {
|
|
245
|
+
const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
|
|
246
|
+
rerankMs = Date.now() - tRerank0;
|
|
247
|
+
this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
|
|
248
|
+
// Cap at `limit`: some rerank servers ignore `top_n` and echo back every
|
|
249
|
+
// candidate, so slicing here keeps the result count within the request.
|
|
250
|
+
results = indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
|
|
251
|
+
}
|
|
252
|
+
catch (err) {
|
|
253
|
+
rerankMs = Date.now() - tRerank0;
|
|
254
|
+
this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
|
|
255
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
256
|
+
ms: Date.now() - t0,
|
|
257
|
+
});
|
|
258
|
+
results = candidates.slice(0, limit);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (metricsOn) {
|
|
262
|
+
this.emitSearchMetrics(query, {
|
|
263
|
+
fetchLimit, candidates, results, poolSaturated, embedMs, dbMs, rerankMs, totalMs: Date.now() - t0,
|
|
191
264
|
});
|
|
192
|
-
return candidates.slice(0, limit);
|
|
193
265
|
}
|
|
266
|
+
return results;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Emit one structured search-tuning line (via `logger.metric`, at `info`
|
|
270
|
+
* regardless of `logging.level`). `candidates` are in cosine order, so a
|
|
271
|
+
* candidate's array index is its cosine rank — `maxCosineRankSurvived` is the
|
|
272
|
+
* deepest cosine position that survived reranking into the returned set. Read
|
|
273
|
+
* it alongside `poolSaturated` to tune `candidateFloor`/`overfetch`/threshold
|
|
274
|
+
* per hardware. Never logs chunk content, embeddings, or credentials.
|
|
275
|
+
*/
|
|
276
|
+
emitSearchMetrics(query, m) {
|
|
277
|
+
const { candidates, results } = m;
|
|
278
|
+
const contentIndex = new Map(candidates.map((c, i) => [c, i]));
|
|
279
|
+
const maxCosineRankSurvived = results.reduce((max, r) => Math.max(max, contentIndex.get(r) ?? -1), -1);
|
|
280
|
+
const fileCounts = new Map();
|
|
281
|
+
for (const r of results)
|
|
282
|
+
fileCounts.set(r.filePath, (fileCounts.get(r.filePath) ?? 0) + 1);
|
|
283
|
+
const topFileCount = results.length ? Math.max(...fileCounts.values()) : 0;
|
|
284
|
+
this.logger.metric('search metrics', {
|
|
285
|
+
query: query.slice(0, 120),
|
|
286
|
+
fetchLimit: m.fetchLimit,
|
|
287
|
+
candidatesReturned: candidates.length,
|
|
288
|
+
poolSaturated: m.poolSaturated,
|
|
289
|
+
maxCosineRankSurvived,
|
|
290
|
+
resultCount: results.length,
|
|
291
|
+
distinctFilesInResult: fileCounts.size,
|
|
292
|
+
topFileShare: results.length ? +(topFileCount / results.length).toFixed(3) : 0,
|
|
293
|
+
bestResultScore: results.length ? Math.min(...results.map(r => r.score)) : null,
|
|
294
|
+
worstResultScore: results.length ? Math.max(...results.map(r => r.score)) : null,
|
|
295
|
+
embedMs: m.embedMs,
|
|
296
|
+
rerankMs: m.rerankMs,
|
|
297
|
+
dbMs: m.dbMs,
|
|
298
|
+
totalMs: m.totalMs,
|
|
299
|
+
});
|
|
194
300
|
}
|
|
195
301
|
async fetchNeighbors(results) {
|
|
196
302
|
const t0 = Date.now();
|
|
@@ -602,11 +708,6 @@ class SearcherService {
|
|
|
602
708
|
this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
|
|
603
709
|
return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
|
|
604
710
|
}
|
|
605
|
-
isNonEnglish(text) {
|
|
606
|
-
// Use Unicode script detection: check for CJK, Cyrillic, Arabic, Hebrew, Thai, etc.
|
|
607
|
-
// Excludes common non-ASCII punctuation (smart quotes, em-dashes, accented Latin chars)
|
|
608
|
-
return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Cyrillic}\p{Script=Arabic}\p{Script=Hebrew}\p{Script=Thai}\p{Script=Devanagari}]/u.test(text);
|
|
609
|
-
}
|
|
610
711
|
walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
|
|
611
712
|
let entries;
|
|
612
713
|
try {
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
5
|
+
/**
|
|
6
|
+
* Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
|
|
7
|
+
* of strings, `input_type` (`query` | `document`) is required for best quality,
|
|
8
|
+
* and the optional `output_dimension` requests Matryoshka truncation. Response
|
|
9
|
+
* is `{ data: [{ embedding, index }] }`.
|
|
10
|
+
*/
|
|
11
|
+
export declare class VoyageEmbedder implements IEmbedder {
|
|
12
|
+
private config;
|
|
13
|
+
private logger;
|
|
14
|
+
readonly dimensions: number;
|
|
15
|
+
rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
|
|
16
|
+
private readonly url;
|
|
17
|
+
private readonly doFetch;
|
|
18
|
+
private readonly baseDelayMs?;
|
|
19
|
+
constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
|
|
20
|
+
embed(text: string, inputType?: InputType): Promise<number[]>;
|
|
21
|
+
embedBatch(texts: string[], inputType?: InputType): Promise<number[][]>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.VoyageEmbedder = void 0;
|
|
4
|
+
const LoggerService_1 = require("./LoggerService");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const reranker_1 = require("./reranker");
|
|
7
|
+
const http_retry_1 = require("./http-retry");
|
|
8
|
+
const constants_1 = require("../constants");
|
|
9
|
+
/**
|
|
10
|
+
* Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
|
|
11
|
+
* of strings, `input_type` (`query` | `document`) is required for best quality,
|
|
12
|
+
* and the optional `output_dimension` requests Matryoshka truncation. Response
|
|
13
|
+
* is `{ data: [{ embedding, index }] }`.
|
|
14
|
+
*/
|
|
15
|
+
class VoyageEmbedder {
|
|
16
|
+
config;
|
|
17
|
+
logger;
|
|
18
|
+
dimensions;
|
|
19
|
+
rerank;
|
|
20
|
+
url;
|
|
21
|
+
doFetch;
|
|
22
|
+
baseDelayMs;
|
|
23
|
+
constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.logger = logger;
|
|
26
|
+
const base = config.embedding.baseURL.replace(/\/+$/, '');
|
|
27
|
+
this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
|
|
28
|
+
this.doFetch = deps.fetchImpl ?? fetch;
|
|
29
|
+
this.baseDelayMs = deps.baseDelayMs;
|
|
30
|
+
this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
|
|
31
|
+
this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
|
|
32
|
+
}
|
|
33
|
+
async embed(text, inputType) {
|
|
34
|
+
const [v] = await this.embedBatch([text], inputType);
|
|
35
|
+
return v;
|
|
36
|
+
}
|
|
37
|
+
async embedBatch(texts, inputType = 'document') {
|
|
38
|
+
if (texts.length === 0)
|
|
39
|
+
return [];
|
|
40
|
+
return (0, http_retry_1.withRetry)(async () => {
|
|
41
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
42
|
+
const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
|
|
43
|
+
if (apiKey)
|
|
44
|
+
headers['Authorization'] = `Bearer ${apiKey}`;
|
|
45
|
+
const body = {
|
|
46
|
+
model: this.config.embedding.model,
|
|
47
|
+
input: texts,
|
|
48
|
+
input_type: inputType,
|
|
49
|
+
};
|
|
50
|
+
if (this.config.embedding.dimensions != null)
|
|
51
|
+
body.output_dimension = this.config.embedding.dimensions;
|
|
52
|
+
const res = await this.doFetch(this.url, {
|
|
53
|
+
method: 'POST',
|
|
54
|
+
headers,
|
|
55
|
+
signal: AbortSignal.timeout(120_000),
|
|
56
|
+
body: JSON.stringify(body),
|
|
57
|
+
});
|
|
58
|
+
if (!res.ok) {
|
|
59
|
+
const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
|
|
60
|
+
throw new http_retry_1.HttpError(res.status, `Voyage embeddings request failed: ${res.status} ${errBody}`);
|
|
61
|
+
}
|
|
62
|
+
const data = await res.json();
|
|
63
|
+
if (!Array.isArray(data.data) || data.data.length !== texts.length) {
|
|
64
|
+
throw new Error(`Voyage embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
|
|
65
|
+
}
|
|
66
|
+
return [...data.data]
|
|
67
|
+
.sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
|
|
68
|
+
.map(d => d.embedding);
|
|
69
|
+
}, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'voyage-embed' });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
exports.VoyageEmbedder = VoyageEmbedder;
|
|
@@ -1,31 +1,55 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Config } from '../config';
|
|
2
2
|
import type { ISearcher } from '../interfaces';
|
|
3
3
|
import type { IEmbedder } from '../interfaces';
|
|
4
4
|
import type { ILogger } from '../interfaces';
|
|
5
5
|
import { type JsonConfig } from '../config-store';
|
|
6
|
+
import { type ProvisioningDeps } from './infinity-provisioning';
|
|
7
|
+
export type BackendActionKind = 'started' | 'already-running' | 'not-needed' | 'manual-required' | 'failed';
|
|
8
|
+
export interface BackendActionResult {
|
|
9
|
+
action: BackendActionKind;
|
|
10
|
+
message: string | null;
|
|
11
|
+
}
|
|
12
|
+
export type BackendStatusState = 'idle' | 'recreating' | 'ready' | 'failed';
|
|
13
|
+
export interface BackendStatus {
|
|
14
|
+
state: BackendStatusState;
|
|
15
|
+
message: string | null;
|
|
16
|
+
startedAt: string | null;
|
|
17
|
+
finishedAt: string | null;
|
|
18
|
+
}
|
|
19
|
+
export interface WebServerDeps {
|
|
20
|
+
/** Path to the managed docker-compose.yml. Defaults to `CONFIG_DIR/docker-compose.yml`. */
|
|
21
|
+
composePath?: string;
|
|
22
|
+
/** Injection seams (fetchImpl/runCommand/pollIntervalMs/readinessTimeoutMs) for the readiness primitive. */
|
|
23
|
+
provisioning?: ProvisioningDeps;
|
|
24
|
+
/** Reads the just-saved config back — defaults to `() => new Config()`. Injectable for tests. */
|
|
25
|
+
readSavedConfig?: () => Config;
|
|
26
|
+
}
|
|
6
27
|
/**
|
|
7
28
|
* Per-field choice lists for the config form. `strict` → render a dropdown that
|
|
8
29
|
* only accepts a listed value (e.g. the embedding model, tied to the vector
|
|
9
30
|
* dimension). Non-strict → a combobox: the values are suggestions but free text
|
|
10
|
-
* is still allowed (e.g. any
|
|
31
|
+
* is still allowed (e.g. any model served by a self-hosted backend).
|
|
11
32
|
*/
|
|
12
33
|
export interface FieldOption {
|
|
13
34
|
values: string[];
|
|
14
35
|
strict: boolean;
|
|
15
36
|
}
|
|
16
|
-
/** Placeholder shown to the browser instead of
|
|
37
|
+
/** Placeholder shown to the browser instead of a real secret. */
|
|
17
38
|
export declare const PASSWORD_MASK = "********";
|
|
18
39
|
/**
|
|
19
|
-
* Merge a config submitted from the browser with the stored one.
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* placeholder.
|
|
40
|
+
* Merge a config submitted from the browser with the stored one. Any secret
|
|
41
|
+
* field (DB password, embedding/reranker API keys) still holding the mask means
|
|
42
|
+
* the user did not change it — restore the stored value rather than overwrite it
|
|
43
|
+
* with the placeholder. `env:` references are not masked and round-trip as-is.
|
|
44
|
+
* Pure — no I/O. Exported for unit testing.
|
|
23
45
|
*/
|
|
24
46
|
export declare function mergeConfigForSave(incoming: JsonConfig, existing: JsonConfig | null): JsonConfig;
|
|
25
47
|
/**
|
|
26
|
-
* Return a copy of the config with
|
|
27
|
-
* real
|
|
28
|
-
*
|
|
48
|
+
* Return a copy of the config with raw secrets (DB password, raw API keys)
|
|
49
|
+
* replaced by the mask, so real secrets never reach the browser. `env:`
|
|
50
|
+
* references are left verbatim — they are not secret and the dashboard shows
|
|
51
|
+
* them as "set via <VAR>". The masked value round-trips through
|
|
52
|
+
* `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
|
|
29
53
|
*/
|
|
30
54
|
export declare function maskConfig(cfg: JsonConfig): JsonConfig;
|
|
31
55
|
/**
|
|
@@ -38,7 +62,13 @@ export declare class WebServer {
|
|
|
38
62
|
private readonly logger;
|
|
39
63
|
private readonly embedder;
|
|
40
64
|
private server?;
|
|
41
|
-
|
|
65
|
+
private readonly composePath;
|
|
66
|
+
private readonly provisioningDeps;
|
|
67
|
+
private readonly readSavedConfig;
|
|
68
|
+
private backendStatus;
|
|
69
|
+
private taskInFlight;
|
|
70
|
+
private pending;
|
|
71
|
+
constructor(searcher: ISearcher, config: Config, logger: ILogger, embedder: IEmbedder, deps?: WebServerDeps);
|
|
42
72
|
start(port: number, host: string): Promise<void>;
|
|
43
73
|
stop(): Promise<void>;
|
|
44
74
|
/** The bound address once started (e.g. `.port` after listen(0)); null otherwise. */
|
|
@@ -49,9 +79,22 @@ export declare class WebServer {
|
|
|
49
79
|
private maskedConfig;
|
|
50
80
|
private saveConfig;
|
|
51
81
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* from the
|
|
82
|
+
* After a successful config write, decide the Infinity backend action per
|
|
83
|
+
* the first-match decision table (`local-backend-provisioning`). Computes
|
|
84
|
+
* the required set from the SAVED config (a fresh `Config` re-read from the
|
|
85
|
+
* file just written — never the server's startup config/embedder). Never
|
|
86
|
+
* throws — an unexpected error becomes `{ action: 'failed', message }`; the
|
|
87
|
+
* saved config and `GET /api/backend-status` are unaffected either way.
|
|
88
|
+
*/
|
|
89
|
+
private evaluateBackendAfterSave;
|
|
90
|
+
/** Kick off (or continue, via a follow-up) a background Infinity recreate. */
|
|
91
|
+
private startBackgroundTask;
|
|
92
|
+
/**
|
|
93
|
+
* Choice lists for model fields. Both are non-strict comboboxes (free text
|
|
94
|
+
* allowed): the embedding model suggests the known registry models plus any
|
|
95
|
+
* locally-installed models; the reranker model suggests installed models.
|
|
96
|
+
* Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
|
|
97
|
+
* still work — their dimension is probed at index time.
|
|
55
98
|
*/
|
|
56
99
|
private fieldOptions;
|
|
57
100
|
private installedModels;
|
|
@@ -38,39 +38,62 @@ exports.mergeConfigForSave = mergeConfigForSave;
|
|
|
38
38
|
exports.maskConfig = maskConfig;
|
|
39
39
|
const http = __importStar(require("http"));
|
|
40
40
|
const path = __importStar(require("path"));
|
|
41
|
+
const config_1 = require("../config");
|
|
41
42
|
const config_store_1 = require("../config-store");
|
|
42
43
|
const constants_1 = require("../constants");
|
|
43
44
|
const graph_1 = require("../graph");
|
|
44
45
|
const dashboard_1 = require("../web/dashboard");
|
|
45
|
-
|
|
46
|
+
const infinity_provisioning_1 = require("./infinity-provisioning");
|
|
47
|
+
function sameModelSet(a, b) {
|
|
48
|
+
return a.join(';') === b.join(';');
|
|
49
|
+
}
|
|
50
|
+
/** Placeholder shown to the browser instead of a real secret. */
|
|
46
51
|
exports.PASSWORD_MASK = '********';
|
|
47
52
|
const MAX_BODY = 1_000_000; // 1 MB — config payloads are tiny
|
|
53
|
+
/** A secret worth masking: a non-empty string that is not an `env:` reference. */
|
|
54
|
+
function isRawSecret(v) {
|
|
55
|
+
return typeof v === 'string' && v !== '' && !v.startsWith('env:');
|
|
56
|
+
}
|
|
48
57
|
/**
|
|
49
|
-
* Merge a config submitted from the browser with the stored one.
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* placeholder.
|
|
58
|
+
* Merge a config submitted from the browser with the stored one. Any secret
|
|
59
|
+
* field (DB password, embedding/reranker API keys) still holding the mask means
|
|
60
|
+
* the user did not change it — restore the stored value rather than overwrite it
|
|
61
|
+
* with the placeholder. `env:` references are not masked and round-trip as-is.
|
|
62
|
+
* Pure — no I/O. Exported for unit testing.
|
|
53
63
|
*/
|
|
54
64
|
function mergeConfigForSave(incoming, existing) {
|
|
55
65
|
const merged = JSON.parse(JSON.stringify(incoming ?? {}));
|
|
56
|
-
|
|
57
|
-
const
|
|
58
|
-
if (
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
66
|
+
const restore = (grp, field) => {
|
|
67
|
+
const g = merged[grp];
|
|
68
|
+
if (g && g[field] === exports.PASSWORD_MASK) {
|
|
69
|
+
const prev = existing?.[grp]?.[field];
|
|
70
|
+
if (prev !== undefined)
|
|
71
|
+
g[field] = prev;
|
|
72
|
+
else
|
|
73
|
+
delete g[field];
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
restore('db', 'password');
|
|
77
|
+
restore('embedding', 'apiKey');
|
|
78
|
+
restore('reranker', 'apiKey');
|
|
63
79
|
return merged;
|
|
64
80
|
}
|
|
65
81
|
/**
|
|
66
|
-
* Return a copy of the config with
|
|
67
|
-
* real
|
|
68
|
-
*
|
|
82
|
+
* Return a copy of the config with raw secrets (DB password, raw API keys)
|
|
83
|
+
* replaced by the mask, so real secrets never reach the browser. `env:`
|
|
84
|
+
* references are left verbatim — they are not secret and the dashboard shows
|
|
85
|
+
* them as "set via <VAR>". The masked value round-trips through
|
|
86
|
+
* `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
|
|
69
87
|
*/
|
|
70
88
|
function maskConfig(cfg) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
89
|
+
const clone = JSON.parse(JSON.stringify(cfg ?? {}));
|
|
90
|
+
if (clone.db?.password)
|
|
91
|
+
clone.db.password = exports.PASSWORD_MASK;
|
|
92
|
+
if (isRawSecret(clone.embedding?.apiKey))
|
|
93
|
+
clone.embedding.apiKey = exports.PASSWORD_MASK;
|
|
94
|
+
if (isRawSecret(clone.reranker?.apiKey))
|
|
95
|
+
clone.reranker.apiKey = exports.PASSWORD_MASK;
|
|
96
|
+
return clone;
|
|
74
97
|
}
|
|
75
98
|
/**
|
|
76
99
|
* Local HTTP dashboard. Serves a schema-driven config editor, a project
|
|
@@ -82,11 +105,22 @@ class WebServer {
|
|
|
82
105
|
logger;
|
|
83
106
|
embedder;
|
|
84
107
|
server;
|
|
85
|
-
|
|
108
|
+
composePath;
|
|
109
|
+
provisioningDeps;
|
|
110
|
+
readSavedConfig;
|
|
111
|
+
// Background Infinity-recreate task state (D6). In-memory, per-process — see
|
|
112
|
+
// `GET /api/backend-status`. `pending` holds at most one queued follow-up.
|
|
113
|
+
backendStatus = { state: 'idle', message: null, startedAt: null, finishedAt: null };
|
|
114
|
+
taskInFlight = false;
|
|
115
|
+
pending = null;
|
|
116
|
+
constructor(searcher, config, logger, embedder, deps = {}) {
|
|
86
117
|
this.searcher = searcher;
|
|
87
118
|
this.config = config;
|
|
88
119
|
this.logger = logger;
|
|
89
120
|
this.embedder = embedder;
|
|
121
|
+
this.composePath = deps.composePath ?? path.join(config_store_1.CONFIG_DIR, 'docker-compose.yml');
|
|
122
|
+
this.provisioningDeps = deps.provisioning ?? {};
|
|
123
|
+
this.readSavedConfig = deps.readSavedConfig ?? (() => new config_1.Config());
|
|
90
124
|
}
|
|
91
125
|
async start(port, host) {
|
|
92
126
|
this.server = http.createServer((req, res) => {
|
|
@@ -131,6 +165,8 @@ class WebServer {
|
|
|
131
165
|
return this.json(res, 200, this.maskedConfig());
|
|
132
166
|
if (m === 'PUT' && p === '/api/config')
|
|
133
167
|
return this.saveConfig(req, res);
|
|
168
|
+
if (m === 'GET' && p === '/api/backend-status')
|
|
169
|
+
return this.json(res, 200, this.backendStatus);
|
|
134
170
|
if (m === 'GET' && p === '/api/projects')
|
|
135
171
|
return this.json(res, 200, await this.searcher.listProjects());
|
|
136
172
|
if (m === 'GET' && p === '/api/graph')
|
|
@@ -168,18 +204,90 @@ class WebServer {
|
|
|
168
204
|
}
|
|
169
205
|
(0, config_store_1.writeConfigJson)(validated);
|
|
170
206
|
this.logger.info('WebServer: config updated');
|
|
171
|
-
|
|
207
|
+
const backend = await this.evaluateBackendAfterSave(validated);
|
|
208
|
+
this.json(res, 200, { ok: true, backend });
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* After a successful config write, decide the Infinity backend action per
|
|
212
|
+
* the first-match decision table (`local-backend-provisioning`). Computes
|
|
213
|
+
* the required set from the SAVED config (a fresh `Config` re-read from the
|
|
214
|
+
* file just written — never the server's startup config/embedder). Never
|
|
215
|
+
* throws — an unexpected error becomes `{ action: 'failed', message }`; the
|
|
216
|
+
* saved config and `GET /api/backend-status` are unaffected either way.
|
|
217
|
+
*/
|
|
218
|
+
async evaluateBackendAfterSave(saved) {
|
|
219
|
+
const provider = saved.embedding?.provider ?? config_store_1.CONFIG_DEFAULTS['embedding.provider'];
|
|
220
|
+
if (provider !== 'infinity') {
|
|
221
|
+
this.pending = null;
|
|
222
|
+
return { action: 'not-needed', message: null };
|
|
223
|
+
}
|
|
224
|
+
try {
|
|
225
|
+
const freshConfig = this.readSavedConfig();
|
|
226
|
+
const baseURL = freshConfig.embedding.baseURL;
|
|
227
|
+
const composePath = this.composePath;
|
|
228
|
+
const modelSet = (0, infinity_provisioning_1.requiredModelSet)(freshConfig);
|
|
229
|
+
const managed = (0, infinity_provisioning_1.isManagedInfinity)(baseURL, composePath);
|
|
230
|
+
if (this.taskInFlight) {
|
|
231
|
+
if (managed) {
|
|
232
|
+
this.pending = { modelSet, baseURL, composePath };
|
|
233
|
+
return { action: 'already-running', message: 'A backend recreate is already in progress; this configuration will be applied next.' };
|
|
234
|
+
}
|
|
235
|
+
// No longer targets the managed container the in-flight task owns.
|
|
236
|
+
this.pending = null;
|
|
237
|
+
}
|
|
238
|
+
const evaluation = await (0, infinity_provisioning_1.evaluateBackend)(baseURL, composePath, modelSet, this.provisioningDeps);
|
|
239
|
+
switch (evaluation.action) {
|
|
240
|
+
case 'served':
|
|
241
|
+
return { action: 'not-needed', message: null };
|
|
242
|
+
case 'missing-external':
|
|
243
|
+
case 'unreachable-external':
|
|
244
|
+
return { action: 'manual-required', message: (0, infinity_provisioning_1.manualInstructionMessage)(evaluation, (0, infinity_provisioning_1.serverRoot)(baseURL), modelSet) };
|
|
245
|
+
case 'missing-managed':
|
|
246
|
+
case 'unreachable-managed':
|
|
247
|
+
this.startBackgroundTask(baseURL, composePath, modelSet);
|
|
248
|
+
return { action: 'started', message: 'Recreating the managed Infinity container in the background.' };
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (err) {
|
|
252
|
+
return { action: 'failed', message: err instanceof Error ? err.message : String(err) };
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/** Kick off (or continue, via a follow-up) a background Infinity recreate. */
|
|
256
|
+
startBackgroundTask(baseURL, composePath, modelSet) {
|
|
257
|
+
this.taskInFlight = true;
|
|
258
|
+
this.backendStatus = { state: 'recreating', message: null, startedAt: new Date().toISOString(), finishedAt: null };
|
|
259
|
+
(0, infinity_provisioning_1.ensureModelSetServed)(baseURL, composePath, modelSet, this.provisioningDeps)
|
|
260
|
+
.then(() => {
|
|
261
|
+
this.backendStatus = { ...this.backendStatus, state: 'ready', message: null, finishedAt: new Date().toISOString() };
|
|
262
|
+
})
|
|
263
|
+
.catch((err) => {
|
|
264
|
+
this.backendStatus = {
|
|
265
|
+
...this.backendStatus, state: 'failed',
|
|
266
|
+
message: err instanceof Error ? err.message : String(err),
|
|
267
|
+
finishedAt: new Date().toISOString(),
|
|
268
|
+
};
|
|
269
|
+
})
|
|
270
|
+
.finally(() => {
|
|
271
|
+
this.taskInFlight = false;
|
|
272
|
+
const next = this.pending;
|
|
273
|
+
this.pending = null;
|
|
274
|
+
if (next && !sameModelSet(next.modelSet, modelSet)) {
|
|
275
|
+
this.startBackgroundTask(next.baseURL, next.composePath, next.modelSet);
|
|
276
|
+
}
|
|
277
|
+
});
|
|
172
278
|
}
|
|
173
279
|
/**
|
|
174
|
-
* Choice lists for model fields.
|
|
175
|
-
*
|
|
176
|
-
*
|
|
280
|
+
* Choice lists for model fields. Both are non-strict comboboxes (free text
|
|
281
|
+
* allowed): the embedding model suggests the known registry models plus any
|
|
282
|
+
* locally-installed models; the reranker model suggests installed models.
|
|
283
|
+
* Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
|
|
284
|
+
* still work — their dimension is probed at index time.
|
|
177
285
|
*/
|
|
178
286
|
async fieldOptions() {
|
|
179
287
|
const installed = await this.installedModels();
|
|
288
|
+
const registryModels = Object.keys(constants_1.MODEL_REGISTRY);
|
|
180
289
|
return {
|
|
181
|
-
'
|
|
182
|
-
'search.translateModel': { values: installed, strict: false },
|
|
290
|
+
'embedding.model': { values: [...new Set([...registryModels, ...installed])], strict: false },
|
|
183
291
|
'reranker.model': { values: installed, strict: false },
|
|
184
292
|
};
|
|
185
293
|
}
|
|
@@ -188,7 +296,7 @@ class WebServer {
|
|
|
188
296
|
return (await this.embedder.listModels?.()) ?? [];
|
|
189
297
|
}
|
|
190
298
|
catch (err) {
|
|
191
|
-
this.logger.warn('WebServer: could not list
|
|
299
|
+
this.logger.warn('WebServer: could not list models', { reason: err instanceof Error ? err.message : String(err) });
|
|
192
300
|
return [];
|
|
193
301
|
}
|
|
194
302
|
}
|