codesearch-mcp 0.0.18-alpha → 0.2.0-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/CHANGELOG.md +47 -1
  2. package/README.md +194 -23
  3. package/dist/app.js +38 -2
  4. package/dist/src/commands/IndexCommand.js +55 -1
  5. package/dist/src/commands/McpCommand.js +6 -2
  6. package/dist/src/config-error.d.ts +7 -0
  7. package/dist/src/config-error.js +14 -0
  8. package/dist/src/config-store.d.ts +73 -17
  9. package/dist/src/config-store.js +142 -27
  10. package/dist/src/config.d.ts +36 -11
  11. package/dist/src/config.js +89 -27
  12. package/dist/src/constants.d.ts +42 -2
  13. package/dist/src/constants.js +61 -7
  14. package/dist/src/container.js +5 -3
  15. package/dist/src/db/index.d.ts +1 -1
  16. package/dist/src/db/index.js +2 -1
  17. package/dist/src/db/schema.d.ts +111 -0
  18. package/dist/src/db/schema.js +15 -1
  19. package/dist/src/help.js +6 -5
  20. package/dist/src/interfaces/IAstParser.d.ts +14 -0
  21. package/dist/src/interfaces/IChunker.d.ts +24 -0
  22. package/dist/src/interfaces/IChunker.js +2 -0
  23. package/dist/src/interfaces/IDatabase.d.ts +31 -0
  24. package/dist/src/interfaces/IEmbedder.d.ts +14 -2
  25. package/dist/src/interfaces/ILogger.d.ts +7 -0
  26. package/dist/src/interfaces/ISearcher.d.ts +1 -0
  27. package/dist/src/interfaces/index.d.ts +2 -1
  28. package/dist/src/owner-path.d.ts +10 -0
  29. package/dist/src/owner-path.js +27 -0
  30. package/dist/src/services/AstChunker.d.ts +42 -0
  31. package/dist/src/services/AstChunker.js +158 -0
  32. package/dist/src/services/AstParserService.d.ts +6 -0
  33. package/dist/src/services/AstParserService.js +31 -2
  34. package/dist/src/services/DatabaseService.d.ts +17 -1
  35. package/dist/src/services/DatabaseService.js +116 -25
  36. package/dist/src/services/EmbedderFactory.d.ts +9 -0
  37. package/dist/src/services/EmbedderFactory.js +26 -0
  38. package/dist/src/services/IndexerService.d.ts +21 -5
  39. package/dist/src/services/IndexerService.js +127 -44
  40. package/dist/src/services/InfinityEmbedder.d.ts +38 -0
  41. package/dist/src/services/InfinityEmbedder.js +88 -0
  42. package/dist/src/services/LoggerService.d.ts +3 -1
  43. package/dist/src/services/LoggerService.js +17 -3
  44. package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
  45. package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
  46. package/dist/src/services/SearcherService.d.ts +21 -1
  47. package/dist/src/services/SearcherService.js +129 -28
  48. package/dist/src/services/VoyageEmbedder.d.ts +22 -0
  49. package/dist/src/services/VoyageEmbedder.js +72 -0
  50. package/dist/src/services/WebServerService.d.ts +57 -14
  51. package/dist/src/services/WebServerService.js +134 -26
  52. package/dist/src/services/http-retry.d.ts +19 -0
  53. package/dist/src/services/http-retry.js +54 -0
  54. package/dist/src/services/infinity-provisioning.d.ts +115 -0
  55. package/dist/src/services/infinity-provisioning.js +516 -0
  56. package/dist/src/services/legacy-ollama-index.d.ts +24 -0
  57. package/dist/src/services/legacy-ollama-index.js +61 -0
  58. package/dist/src/services/reranker.d.ts +15 -0
  59. package/dist/src/services/reranker.js +44 -0
  60. package/dist/src/setup.d.ts +43 -0
  61. package/dist/src/setup.js +227 -55
  62. package/dist/src/tools/SearchCodebaseTool.js +1 -1
  63. package/dist/src/tools/format-helpers.js +2 -1
  64. package/dist/src/web/dashboard.js +5 -0
  65. package/docker/docker-compose.infinity-gpu.yml +19 -0
  66. package/docker/docker-compose.yml +20 -5
  67. package/docs/web-dashboard.md +36 -17
  68. package/package.json +18 -3
  69. package/dist/src/ast.d.ts +0 -2
  70. package/dist/src/ast.js +0 -110
  71. package/dist/src/commands/SetupCommand.d.ts +0 -7
  72. package/dist/src/commands/SetupCommand.js +0 -14
  73. package/dist/src/db/data-source.d.ts +0 -2
  74. package/dist/src/db/data-source.js +0 -13
  75. package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
  76. package/dist/src/db/entities/code-chunk.entity.js +0 -54
  77. package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
  78. package/dist/src/db/entities/code-dependency.entity.js +0 -54
  79. package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
  80. package/dist/src/db/entities/code-symbol.entity.js +0 -59
  81. package/dist/src/db/init.d.ts +0 -2
  82. package/dist/src/db/init.js +0 -62
  83. package/dist/src/env-loader.d.ts +0 -1
  84. package/dist/src/env-loader.js +0 -41
  85. package/dist/src/indexer.d.ts +0 -17
  86. package/dist/src/indexer.js +0 -242
  87. package/dist/src/mcp.d.ts +0 -1
  88. package/dist/src/mcp.js +0 -403
  89. package/dist/src/ollama.d.ts +0 -2
  90. package/dist/src/ollama.js +0 -6
  91. package/dist/src/searcher.d.ts +0 -74
  92. package/dist/src/searcher.js +0 -446
  93. package/dist/src/services/OllamaEmbedder.d.ts +0 -14
  94. package/dist/src/services/OllamaEmbedder.js +0 -63
  95. package/dist/vitest.config.d.ts +0 -2
  96. package/dist/vitest.config.js +0 -9
@@ -40,11 +40,13 @@ const readline = __importStar(require("readline"));
40
40
  const child_process_1 = require("child_process");
41
41
  const file_utils_1 = require("../file-utils");
42
42
  const MonorepoAwareDiscoveryService_1 = require("./MonorepoAwareDiscoveryService");
43
+ const legacy_ollama_index_1 = require("./legacy-ollama-index");
43
44
  const SILENT_LOGGER = {
44
45
  debug() { },
45
46
  info() { },
46
47
  warn() { },
47
48
  error() { },
49
+ metric() { },
48
50
  };
49
51
  const EMPTY_META = { description: null, stack: null, notes: null, relations: null };
50
52
  const GREP_MAX_RESULTS = 200;
@@ -127,16 +129,53 @@ class SearcherService {
127
129
  }
128
130
  return edges;
129
131
  }
132
+ // Query translation was removed (legacy). The query is embedded as-is; the
133
+ // method is kept so the tool layer needs no change and `translated` is always
134
+ // false.
130
135
  async prepareQuery(raw) {
131
- const t0 = Date.now();
132
- if (this.config.search.autoTranslate && this.isNonEnglish(raw) && this.embedder.translate) {
133
- const query = await this.embedder.translate(raw, this.config.search.translateModel);
134
- this.logger.debug('SearcherService: prepareQuery', { translated: true, raw: raw.slice(0, 120), query: query.slice(0, 120), ms: Date.now() - t0 });
135
- return { query, translated: true };
136
- }
137
- this.logger.debug('SearcherService: prepareQuery', { translated: false, query: raw.slice(0, 120), ms: Date.now() - t0 });
138
136
  return { query: raw, translated: false };
139
137
  }
138
+ /**
139
+ * Refuse when the configured embedding model disagrees with the one that built
140
+ * the index — embedding a query into a foreign vector space returns noise. A
141
+ * missing `index_meta` row with an empty (or altogether absent) `code_chunk`
142
+ * does not block — but a *populated* `code_chunk` with no `index_meta` row is a
143
+ * pre-`index_meta` install, treated as the legacy `ollama`/`bge-m3` index (same
144
+ * interpretation as the `rag-index` legacy-Ollama guard), read-only — this check
145
+ * never writes `index_meta`.
146
+ */
147
+ async assertIndexMatches() {
148
+ const meta = await this.db.readIndexMeta();
149
+ if (meta) {
150
+ this.compareIndexed(meta.provider, meta.model, meta.dimensions);
151
+ return;
152
+ }
153
+ // No row — a missing code_chunk table counts as empty; any other DB error
154
+ // while counting propagates rather than being read as "not indexed".
155
+ if (await (0, legacy_ollama_index_1.hasIndexedChunks)(this.db)) {
156
+ this.compareIndexed(legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.provider, legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.model, null);
157
+ }
158
+ }
159
+ compareIndexed(indexedProvider, indexedModel, indexedDimensions) {
160
+ const { provider, model } = this.config.embedding;
161
+ if (indexedProvider !== provider || indexedModel !== model) {
162
+ throw new Error(`Index was built with ${indexedProvider}/${indexedModel} but the config selects ${provider}/${model}. ` +
163
+ `Run \`rag-index --reindex\` to rebuild the index for the new model.`);
164
+ }
165
+ // Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
166
+ // override) would embed the query at a width the column can't hold — pgvector
167
+ // would throw an opaque "different vector dimensions" error mid-query. Catch
168
+ // it here with the friendly hint. `embedder.dimensions` is 0 when the width is
169
+ // only knowable by probing, in which case we skip (can't compare cheaply).
170
+ // `indexedDimensions` is null for the legacy-inferred identity, whose provider
171
+ // can never match a configured provider (ollama is no longer selectable), so
172
+ // this branch is unreached in that case.
173
+ const dim = this.embedder.dimensions;
174
+ if (indexedDimensions != null && dim > 0 && dim !== indexedDimensions) {
175
+ throw new Error(`Index was built at ${indexedDimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
176
+ `Run \`rag-index --reindex\` to rebuild the index at the new dimension.`);
177
+ }
178
+ }
140
179
  async listProjects() {
141
180
  const rows = await this.db.query(`SELECT "serviceName", COUNT(*)::int AS chunks
142
181
  FROM code_chunk
@@ -150,47 +189,114 @@ class SearcherService {
150
189
  }
151
190
  async searchChunks(query, limit, project, threshold) {
152
191
  const t0 = Date.now();
192
+ await this.assertIndexMatches();
153
193
  const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
154
- const fetchLimit = this.embedder.rerank
155
- ? Math.min(limit * this.config.search.rerankerOverfetch, 200)
194
+ const reranking = !!this.embedder.rerank;
195
+ // Candidate floor: decouple the pool size from `limit` so a small `limit`
196
+ // (e.g. the default 5) still gives the reranker a large-enough menu. Without
197
+ // a reranker the extra candidates are never reordered, so keep `limit`.
198
+ const fetchLimit = reranking
199
+ ? Math.min(Math.max(limit * this.config.reranker.overfetch, this.config.reranker.candidateFloor), 200)
156
200
  : limit;
201
+ // With metrics on, fetch one extra row to detect a saturated pool (more
202
+ // under-threshold candidates existed than the window took). The extra row is
203
+ // dropped before reranking so it never affects results.
204
+ const metricsOn = this.config.logging.searchMetrics;
205
+ const dbLimit = metricsOn ? fetchLimit + 1 : fetchLimit;
157
206
  this.logger.debug('SearcherService: searchChunks', {
158
207
  query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
159
208
  });
160
- const embedding = await this.embedder.embed(query);
161
- const params = [`[${embedding}]`, effectiveThreshold, fetchLimit];
209
+ const tEmbed0 = Date.now();
210
+ const embedding = await this.embedder.embed(query, 'query');
211
+ const embedMs = Date.now() - tEmbed0;
212
+ const params = [`[${embedding}]`, effectiveThreshold, dbLimit];
162
213
  let projectFilter = '';
163
214
  if (project) {
164
215
  params.push(project);
165
216
  projectFilter = `AND "serviceName" = $${params.length}`;
166
217
  }
167
- const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content,
218
+ const tDb0 = Date.now();
219
+ const fetched = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
168
220
  ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
169
221
  FROM code_chunk
170
222
  WHERE embedding <=> $1 < $2
171
223
  ${projectFilter}
172
224
  ORDER BY embedding <=> $1
173
225
  LIMIT $3`, params);
226
+ const dbMs = Date.now() - tDb0;
227
+ // Saturated iff the extra probe row came back — more candidates were under
228
+ // the threshold than `fetchLimit`. Trim the probe row before reranking.
229
+ const poolSaturated = metricsOn && fetched.length > fetchLimit;
230
+ const candidates = fetched.slice(0, fetchLimit);
174
231
  this.logger.debug('SearcherService: searchChunks candidates', {
175
232
  count: candidates.length,
176
233
  scores: candidates.map(r => r.score),
177
234
  files: candidates.map(r => r.filePath),
178
235
  ms: Date.now() - t0,
179
236
  });
180
- if (!this.embedder.rerank || candidates.length === 0)
181
- return candidates;
182
- try {
183
- const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
184
- this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
185
- return indices.filter(i => i >= 0 && i < candidates.length).map(i => candidates[i]);
237
+ let results;
238
+ let rerankMs = 0;
239
+ if (!reranking || candidates.length === 0) {
240
+ results = candidates.slice(0, limit);
186
241
  }
187
- catch (err) {
188
- this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
189
- reason: err instanceof Error ? err.message : String(err),
190
- ms: Date.now() - t0,
242
+ else {
243
+ const tRerank0 = Date.now();
244
+ try {
245
+ const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
246
+ rerankMs = Date.now() - tRerank0;
247
+ this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
248
+ // Cap at `limit`: some rerank servers ignore `top_n` and echo back every
249
+ // candidate, so slicing here keeps the result count within the request.
250
+ results = indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
251
+ }
252
+ catch (err) {
253
+ rerankMs = Date.now() - tRerank0;
254
+ this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
255
+ reason: err instanceof Error ? err.message : String(err),
256
+ ms: Date.now() - t0,
257
+ });
258
+ results = candidates.slice(0, limit);
259
+ }
260
+ }
261
+ if (metricsOn) {
262
+ this.emitSearchMetrics(query, {
263
+ fetchLimit, candidates, results, poolSaturated, embedMs, dbMs, rerankMs, totalMs: Date.now() - t0,
191
264
  });
192
- return candidates.slice(0, limit);
193
265
  }
266
+ return results;
267
+ }
268
+ /**
269
+ * Emit one structured search-tuning line (via `logger.metric`, at `info`
270
+ * regardless of `logging.level`). `candidates` are in cosine order, so a
271
+ * candidate's array index is its cosine rank — `maxCosineRankSurvived` is the
272
+ * deepest cosine position that survived reranking into the returned set. Read
273
+ * it alongside `poolSaturated` to tune `candidateFloor`/`overfetch`/threshold
274
+ * per hardware. Never logs chunk content, embeddings, or credentials.
275
+ */
276
+ emitSearchMetrics(query, m) {
277
+ const { candidates, results } = m;
278
+ const contentIndex = new Map(candidates.map((c, i) => [c, i]));
279
+ const maxCosineRankSurvived = results.reduce((max, r) => Math.max(max, contentIndex.get(r) ?? -1), -1);
280
+ const fileCounts = new Map();
281
+ for (const r of results)
282
+ fileCounts.set(r.filePath, (fileCounts.get(r.filePath) ?? 0) + 1);
283
+ const topFileCount = results.length ? Math.max(...fileCounts.values()) : 0;
284
+ this.logger.metric('search metrics', {
285
+ query: query.slice(0, 120),
286
+ fetchLimit: m.fetchLimit,
287
+ candidatesReturned: candidates.length,
288
+ poolSaturated: m.poolSaturated,
289
+ maxCosineRankSurvived,
290
+ resultCount: results.length,
291
+ distinctFilesInResult: fileCounts.size,
292
+ topFileShare: results.length ? +(topFileCount / results.length).toFixed(3) : 0,
293
+ bestResultScore: results.length ? Math.min(...results.map(r => r.score)) : null,
294
+ worstResultScore: results.length ? Math.max(...results.map(r => r.score)) : null,
295
+ embedMs: m.embedMs,
296
+ rerankMs: m.rerankMs,
297
+ dbMs: m.dbMs,
298
+ totalMs: m.totalMs,
299
+ });
194
300
  }
195
301
  async fetchNeighbors(results) {
196
302
  const t0 = Date.now();
@@ -602,11 +708,6 @@ class SearcherService {
602
708
  this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
603
709
  return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
604
710
  }
605
- isNonEnglish(text) {
606
- // Use Unicode script detection: check for CJK, Cyrillic, Arabic, Hebrew, Thai, etc.
607
- // Excludes common non-ASCII punctuation (smart quotes, em-dashes, accented Latin chars)
608
- return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Cyrillic}\p{Script=Arabic}\p{Script=Hebrew}\p{Script=Thai}\p{Script=Devanagari}]/u.test(text);
609
- }
610
711
  walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
611
712
  let entries;
612
713
  try {
@@ -0,0 +1,22 @@
1
+ import type { Config } from '../config';
2
+ import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
3
+ import type { ILogger } from '../interfaces/ILogger';
4
+ import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
5
+ /**
6
+ * Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
7
+ * of strings, `input_type` (`query` | `document`) is required for best quality,
8
+ * and the optional `output_dimension` requests Matryoshka truncation. Response
9
+ * is `{ data: [{ embedding, index }] }`.
10
+ */
11
+ export declare class VoyageEmbedder implements IEmbedder {
12
+ private config;
13
+ private logger;
14
+ readonly dimensions: number;
15
+ rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
16
+ private readonly url;
17
+ private readonly doFetch;
18
+ private readonly baseDelayMs?;
19
+ constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
20
+ embed(text: string, inputType?: InputType): Promise<number[]>;
21
+ embedBatch(texts: string[], inputType?: InputType): Promise<number[][]>;
22
+ }
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.VoyageEmbedder = void 0;
4
+ const LoggerService_1 = require("./LoggerService");
5
+ const config_1 = require("../config");
6
+ const reranker_1 = require("./reranker");
7
+ const http_retry_1 = require("./http-retry");
8
+ const constants_1 = require("../constants");
9
+ /**
10
+ * Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
11
+ * of strings, `input_type` (`query` | `document`) is required for best quality,
12
+ * and the optional `output_dimension` requests Matryoshka truncation. Response
13
+ * is `{ data: [{ embedding, index }] }`.
14
+ */
15
+ class VoyageEmbedder {
16
+ config;
17
+ logger;
18
+ dimensions;
19
+ rerank;
20
+ url;
21
+ doFetch;
22
+ baseDelayMs;
23
+ constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
24
+ this.config = config;
25
+ this.logger = logger;
26
+ const base = config.embedding.baseURL.replace(/\/+$/, '');
27
+ this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
28
+ this.doFetch = deps.fetchImpl ?? fetch;
29
+ this.baseDelayMs = deps.baseDelayMs;
30
+ this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
31
+ this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
32
+ }
33
+ async embed(text, inputType) {
34
+ const [v] = await this.embedBatch([text], inputType);
35
+ return v;
36
+ }
37
+ async embedBatch(texts, inputType = 'document') {
38
+ if (texts.length === 0)
39
+ return [];
40
+ return (0, http_retry_1.withRetry)(async () => {
41
+ const headers = { 'Content-Type': 'application/json' };
42
+ const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
43
+ if (apiKey)
44
+ headers['Authorization'] = `Bearer ${apiKey}`;
45
+ const body = {
46
+ model: this.config.embedding.model,
47
+ input: texts,
48
+ input_type: inputType,
49
+ };
50
+ if (this.config.embedding.dimensions != null)
51
+ body.output_dimension = this.config.embedding.dimensions;
52
+ const res = await this.doFetch(this.url, {
53
+ method: 'POST',
54
+ headers,
55
+ signal: AbortSignal.timeout(120_000),
56
+ body: JSON.stringify(body),
57
+ });
58
+ if (!res.ok) {
59
+ const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
60
+ throw new http_retry_1.HttpError(res.status, `Voyage embeddings request failed: ${res.status} ${errBody}`);
61
+ }
62
+ const data = await res.json();
63
+ if (!Array.isArray(data.data) || data.data.length !== texts.length) {
64
+ throw new Error(`Voyage embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
65
+ }
66
+ return [...data.data]
67
+ .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
68
+ .map(d => d.embedding);
69
+ }, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'voyage-embed' });
70
+ }
71
+ }
72
+ exports.VoyageEmbedder = VoyageEmbedder;
@@ -1,31 +1,55 @@
1
- import type { Config } from '../config';
1
+ import { Config } from '../config';
2
2
  import type { ISearcher } from '../interfaces';
3
3
  import type { IEmbedder } from '../interfaces';
4
4
  import type { ILogger } from '../interfaces';
5
5
  import { type JsonConfig } from '../config-store';
6
+ import { type ProvisioningDeps } from './infinity-provisioning';
7
+ export type BackendActionKind = 'started' | 'already-running' | 'not-needed' | 'manual-required' | 'failed';
8
+ export interface BackendActionResult {
9
+ action: BackendActionKind;
10
+ message: string | null;
11
+ }
12
+ export type BackendStatusState = 'idle' | 'recreating' | 'ready' | 'failed';
13
+ export interface BackendStatus {
14
+ state: BackendStatusState;
15
+ message: string | null;
16
+ startedAt: string | null;
17
+ finishedAt: string | null;
18
+ }
19
+ export interface WebServerDeps {
20
+ /** Path to the managed docker-compose.yml. Defaults to `CONFIG_DIR/docker-compose.yml`. */
21
+ composePath?: string;
22
+ /** Injection seams (fetchImpl/runCommand/pollIntervalMs/readinessTimeoutMs) for the readiness primitive. */
23
+ provisioning?: ProvisioningDeps;
24
+ /** Reads the just-saved config back — defaults to `() => new Config()`. Injectable for tests. */
25
+ readSavedConfig?: () => Config;
26
+ }
6
27
  /**
7
28
  * Per-field choice lists for the config form. `strict` → render a dropdown that
8
29
  * only accepts a listed value (e.g. the embedding model, tied to the vector
9
30
  * dimension). Non-strict → a combobox: the values are suggestions but free text
10
- * is still allowed (e.g. any Ollama model the user has pulled).
31
+ * is still allowed (e.g. any model served by a self-hosted backend).
11
32
  */
12
33
  export interface FieldOption {
13
34
  values: string[];
14
35
  strict: boolean;
15
36
  }
16
- /** Placeholder shown to the browser instead of the real DB password. */
37
+ /** Placeholder shown to the browser instead of a real secret. */
17
38
  export declare const PASSWORD_MASK = "********";
18
39
  /**
19
- * Merge a config submitted from the browser with the stored one. If the
20
- * password field still holds the mask (i.e. the user did not change it),
21
- * preserve the previously stored password rather than overwriting it with the
22
- * placeholder. Pure — no I/O. Exported for unit testing.
40
+ * Merge a config submitted from the browser with the stored one. Any secret
41
+ * field (DB password, embedding/reranker API keys) still holding the mask means
42
+ * the user did not change it — restore the stored value rather than overwrite it
43
+ * with the placeholder. `env:` references are not masked and round-trip as-is.
44
+ * Pure — no I/O. Exported for unit testing.
23
45
  */
24
46
  export declare function mergeConfigForSave(incoming: JsonConfig, existing: JsonConfig | null): JsonConfig;
25
47
  /**
26
- * Return a copy of the config with the DB password replaced by the mask, so the
27
- * real secret never reaches the browser. The masked value round-trips through
28
- * `mergeConfigForSave` on save. Pure — no I/O. Exported for unit testing.
48
+ * Return a copy of the config with raw secrets (DB password, raw API keys)
49
+ * replaced by the mask, so real secrets never reach the browser. `env:`
50
+ * references are left verbatim — they are not secret and the dashboard shows
51
+ * them as "set via <VAR>". The masked value round-trips through
52
+ * `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
29
53
  */
30
54
  export declare function maskConfig(cfg: JsonConfig): JsonConfig;
31
55
  /**
@@ -38,7 +62,13 @@ export declare class WebServer {
38
62
  private readonly logger;
39
63
  private readonly embedder;
40
64
  private server?;
41
- constructor(searcher: ISearcher, config: Config, logger: ILogger, embedder: IEmbedder);
65
+ private readonly composePath;
66
+ private readonly provisioningDeps;
67
+ private readonly readSavedConfig;
68
+ private backendStatus;
69
+ private taskInFlight;
70
+ private pending;
71
+ constructor(searcher: ISearcher, config: Config, logger: ILogger, embedder: IEmbedder, deps?: WebServerDeps);
42
72
  start(port: number, host: string): Promise<void>;
43
73
  stop(): Promise<void>;
44
74
  /** The bound address once started (e.g. `.port` after listen(0)); null otherwise. */
@@ -49,9 +79,22 @@ export declare class WebServer {
49
79
  private maskedConfig;
50
80
  private saveConfig;
51
81
  /**
52
- * Choice lists for model fields. The embedding model is strict (only
53
- * dimension-compatible models); the translate/reranker models are suggested
54
- * from the user's installed Ollama models but stay free-text.
82
+ * After a successful config write, decide the Infinity backend action per
83
+ * the first-match decision table (`local-backend-provisioning`). Computes
84
+ * the required set from the SAVED config (a fresh `Config` re-read from the
85
+ * file just written — never the server's startup config/embedder). Never
86
+ * throws — an unexpected error becomes `{ action: 'failed', message }`; the
87
+ * saved config and `GET /api/backend-status` are unaffected either way.
88
+ */
89
+ private evaluateBackendAfterSave;
90
+ /** Kick off (or continue, via a follow-up) a background Infinity recreate. */
91
+ private startBackgroundTask;
92
+ /**
93
+ * Choice lists for model fields. Both are non-strict comboboxes (free text
94
+ * allowed): the embedding model suggests the known registry models plus any
95
+ * locally-installed models; the reranker model suggests installed models.
96
+ * Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
97
+ * still work — their dimension is probed at index time.
55
98
  */
56
99
  private fieldOptions;
57
100
  private installedModels;
@@ -38,39 +38,62 @@ exports.mergeConfigForSave = mergeConfigForSave;
38
38
  exports.maskConfig = maskConfig;
39
39
  const http = __importStar(require("http"));
40
40
  const path = __importStar(require("path"));
41
+ const config_1 = require("../config");
41
42
  const config_store_1 = require("../config-store");
42
43
  const constants_1 = require("../constants");
43
44
  const graph_1 = require("../graph");
44
45
  const dashboard_1 = require("../web/dashboard");
45
- /** Placeholder shown to the browser instead of the real DB password. */
46
+ const infinity_provisioning_1 = require("./infinity-provisioning");
47
+ function sameModelSet(a, b) {
48
+ return a.join(';') === b.join(';');
49
+ }
50
+ /** Placeholder shown to the browser instead of a real secret. */
46
51
  exports.PASSWORD_MASK = '********';
47
52
  const MAX_BODY = 1_000_000; // 1 MB — config payloads are tiny
53
+ /** A secret worth masking: a non-empty string that is not an `env:` reference. */
54
+ function isRawSecret(v) {
55
+ return typeof v === 'string' && v !== '' && !v.startsWith('env:');
56
+ }
48
57
  /**
49
- * Merge a config submitted from the browser with the stored one. If the
50
- * password field still holds the mask (i.e. the user did not change it),
51
- * preserve the previously stored password rather than overwriting it with the
52
- * placeholder. Pure — no I/O. Exported for unit testing.
58
+ * Merge a config submitted from the browser with the stored one. Any secret
59
+ * field (DB password, embedding/reranker API keys) still holding the mask means
60
+ * the user did not change it — restore the stored value rather than overwrite it
61
+ * with the placeholder. `env:` references are not masked and round-trip as-is.
62
+ * Pure — no I/O. Exported for unit testing.
53
63
  */
54
64
  function mergeConfigForSave(incoming, existing) {
55
65
  const merged = JSON.parse(JSON.stringify(incoming ?? {}));
56
- if (merged.db && merged.db.password === exports.PASSWORD_MASK) {
57
- const prev = existing?.db?.password;
58
- if (prev !== undefined)
59
- merged.db.password = prev;
60
- else
61
- delete merged.db.password;
62
- }
66
+ const restore = (grp, field) => {
67
+ const g = merged[grp];
68
+ if (g && g[field] === exports.PASSWORD_MASK) {
69
+ const prev = existing?.[grp]?.[field];
70
+ if (prev !== undefined)
71
+ g[field] = prev;
72
+ else
73
+ delete g[field];
74
+ }
75
+ };
76
+ restore('db', 'password');
77
+ restore('embedding', 'apiKey');
78
+ restore('reranker', 'apiKey');
63
79
  return merged;
64
80
  }
65
81
  /**
66
- * Return a copy of the config with the DB password replaced by the mask, so the
67
- * real secret never reaches the browser. The masked value round-trips through
68
- * `mergeConfigForSave` on save. Pure — no I/O. Exported for unit testing.
82
+ * Return a copy of the config with raw secrets (DB password, raw API keys)
83
+ * replaced by the mask, so real secrets never reach the browser. `env:`
84
+ * references are left verbatim — they are not secret and the dashboard shows
85
+ * them as "set via <VAR>". The masked value round-trips through
86
+ * `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
69
87
  */
70
88
  function maskConfig(cfg) {
71
- if (!cfg.db?.password)
72
- return cfg;
73
- return { ...cfg, db: { ...cfg.db, password: exports.PASSWORD_MASK } };
89
+ const clone = JSON.parse(JSON.stringify(cfg ?? {}));
90
+ if (clone.db?.password)
91
+ clone.db.password = exports.PASSWORD_MASK;
92
+ if (isRawSecret(clone.embedding?.apiKey))
93
+ clone.embedding.apiKey = exports.PASSWORD_MASK;
94
+ if (isRawSecret(clone.reranker?.apiKey))
95
+ clone.reranker.apiKey = exports.PASSWORD_MASK;
96
+ return clone;
74
97
  }
75
98
  /**
76
99
  * Local HTTP dashboard. Serves a schema-driven config editor, a project
@@ -82,11 +105,22 @@ class WebServer {
82
105
  logger;
83
106
  embedder;
84
107
  server;
85
- constructor(searcher, config, logger, embedder) {
108
+ composePath;
109
+ provisioningDeps;
110
+ readSavedConfig;
111
+ // Background Infinity-recreate task state (D6). In-memory, per-process — see
112
+ // `GET /api/backend-status`. `pending` holds at most one queued follow-up.
113
+ backendStatus = { state: 'idle', message: null, startedAt: null, finishedAt: null };
114
+ taskInFlight = false;
115
+ pending = null;
116
+ constructor(searcher, config, logger, embedder, deps = {}) {
86
117
  this.searcher = searcher;
87
118
  this.config = config;
88
119
  this.logger = logger;
89
120
  this.embedder = embedder;
121
+ this.composePath = deps.composePath ?? path.join(config_store_1.CONFIG_DIR, 'docker-compose.yml');
122
+ this.provisioningDeps = deps.provisioning ?? {};
123
+ this.readSavedConfig = deps.readSavedConfig ?? (() => new config_1.Config());
90
124
  }
91
125
  async start(port, host) {
92
126
  this.server = http.createServer((req, res) => {
@@ -131,6 +165,8 @@ class WebServer {
131
165
  return this.json(res, 200, this.maskedConfig());
132
166
  if (m === 'PUT' && p === '/api/config')
133
167
  return this.saveConfig(req, res);
168
+ if (m === 'GET' && p === '/api/backend-status')
169
+ return this.json(res, 200, this.backendStatus);
134
170
  if (m === 'GET' && p === '/api/projects')
135
171
  return this.json(res, 200, await this.searcher.listProjects());
136
172
  if (m === 'GET' && p === '/api/graph')
@@ -168,18 +204,90 @@ class WebServer {
168
204
  }
169
205
  (0, config_store_1.writeConfigJson)(validated);
170
206
  this.logger.info('WebServer: config updated');
171
- this.json(res, 200, { ok: true });
207
+ const backend = await this.evaluateBackendAfterSave(validated);
208
+ this.json(res, 200, { ok: true, backend });
209
+ }
210
+ /**
211
+ * After a successful config write, decide the Infinity backend action per
212
+ * the first-match decision table (`local-backend-provisioning`). Computes
213
+ * the required set from the SAVED config (a fresh `Config` re-read from the
214
+ * file just written — never the server's startup config/embedder). Never
215
+ * throws — an unexpected error becomes `{ action: 'failed', message }`; the
216
+ * saved config and `GET /api/backend-status` are unaffected either way.
217
+ */
218
+ async evaluateBackendAfterSave(saved) {
219
+ const provider = saved.embedding?.provider ?? config_store_1.CONFIG_DEFAULTS['embedding.provider'];
220
+ if (provider !== 'infinity') {
221
+ this.pending = null;
222
+ return { action: 'not-needed', message: null };
223
+ }
224
+ try {
225
+ const freshConfig = this.readSavedConfig();
226
+ const baseURL = freshConfig.embedding.baseURL;
227
+ const composePath = this.composePath;
228
+ const modelSet = (0, infinity_provisioning_1.requiredModelSet)(freshConfig);
229
+ const managed = (0, infinity_provisioning_1.isManagedInfinity)(baseURL, composePath);
230
+ if (this.taskInFlight) {
231
+ if (managed) {
232
+ this.pending = { modelSet, baseURL, composePath };
233
+ return { action: 'already-running', message: 'A backend recreate is already in progress; this configuration will be applied next.' };
234
+ }
235
+ // No longer targets the managed container the in-flight task owns.
236
+ this.pending = null;
237
+ }
238
+ const evaluation = await (0, infinity_provisioning_1.evaluateBackend)(baseURL, composePath, modelSet, this.provisioningDeps);
239
+ switch (evaluation.action) {
240
+ case 'served':
241
+ return { action: 'not-needed', message: null };
242
+ case 'missing-external':
243
+ case 'unreachable-external':
244
+ return { action: 'manual-required', message: (0, infinity_provisioning_1.manualInstructionMessage)(evaluation, (0, infinity_provisioning_1.serverRoot)(baseURL), modelSet) };
245
+ case 'missing-managed':
246
+ case 'unreachable-managed':
247
+ this.startBackgroundTask(baseURL, composePath, modelSet);
248
+ return { action: 'started', message: 'Recreating the managed Infinity container in the background.' };
249
+ }
250
+ }
251
+ catch (err) {
252
+ return { action: 'failed', message: err instanceof Error ? err.message : String(err) };
253
+ }
254
+ }
255
+ /** Kick off (or continue, via a follow-up) a background Infinity recreate. */
256
+ startBackgroundTask(baseURL, composePath, modelSet) {
257
+ this.taskInFlight = true;
258
+ this.backendStatus = { state: 'recreating', message: null, startedAt: new Date().toISOString(), finishedAt: null };
259
+ (0, infinity_provisioning_1.ensureModelSetServed)(baseURL, composePath, modelSet, this.provisioningDeps)
260
+ .then(() => {
261
+ this.backendStatus = { ...this.backendStatus, state: 'ready', message: null, finishedAt: new Date().toISOString() };
262
+ })
263
+ .catch((err) => {
264
+ this.backendStatus = {
265
+ ...this.backendStatus, state: 'failed',
266
+ message: err instanceof Error ? err.message : String(err),
267
+ finishedAt: new Date().toISOString(),
268
+ };
269
+ })
270
+ .finally(() => {
271
+ this.taskInFlight = false;
272
+ const next = this.pending;
273
+ this.pending = null;
274
+ if (next && !sameModelSet(next.modelSet, modelSet)) {
275
+ this.startBackgroundTask(next.baseURL, next.composePath, next.modelSet);
276
+ }
277
+ });
172
278
  }
173
279
  /**
174
- * Choice lists for model fields. The embedding model is strict (only
175
- * dimension-compatible models); the translate/reranker models are suggested
176
- * from the user's installed Ollama models but stay free-text.
280
+ * Choice lists for model fields. Both are non-strict comboboxes (free text
281
+ * allowed): the embedding model suggests the known registry models plus any
282
+ * locally-installed models; the reranker model suggests installed models.
283
+ * Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
284
+ * still work — their dimension is probed at index time.
177
285
  */
178
286
  async fieldOptions() {
179
287
  const installed = await this.installedModels();
288
+ const registryModels = Object.keys(constants_1.MODEL_REGISTRY);
180
289
  return {
181
- 'ollama.model': { values: [...constants_1.SUPPORTED_EMBED_MODELS], strict: true },
182
- 'search.translateModel': { values: installed, strict: false },
290
+ 'embedding.model': { values: [...new Set([...registryModels, ...installed])], strict: false },
183
291
  'reranker.model': { values: installed, strict: false },
184
292
  };
185
293
  }
@@ -188,7 +296,7 @@ class WebServer {
188
296
  return (await this.embedder.listModels?.()) ?? [];
189
297
  }
190
298
  catch (err) {
191
- this.logger.warn('WebServer: could not list Ollama models', { reason: err instanceof Error ? err.message : String(err) });
299
+ this.logger.warn('WebServer: could not list models', { reason: err instanceof Error ? err.message : String(err) });
192
300
  return [];
193
301
  }
194
302
  }