codesearch-mcp 0.0.18-alpha → 0.1.0-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +75 -6
  3. package/dist/app.js +36 -0
  4. package/dist/src/commands/IndexCommand.js +14 -1
  5. package/dist/src/commands/McpCommand.js +6 -2
  6. package/dist/src/config-store.d.ts +49 -11
  7. package/dist/src/config-store.js +79 -19
  8. package/dist/src/config.d.ts +32 -8
  9. package/dist/src/config.js +84 -20
  10. package/dist/src/constants.d.ts +36 -2
  11. package/dist/src/constants.js +58 -7
  12. package/dist/src/container.js +5 -3
  13. package/dist/src/db/index.d.ts +1 -1
  14. package/dist/src/db/index.js +2 -1
  15. package/dist/src/db/schema.d.ts +111 -0
  16. package/dist/src/db/schema.js +15 -1
  17. package/dist/src/help.js +5 -4
  18. package/dist/src/interfaces/IAstParser.d.ts +14 -0
  19. package/dist/src/interfaces/IChunker.d.ts +24 -0
  20. package/dist/src/interfaces/IChunker.js +2 -0
  21. package/dist/src/interfaces/IDatabase.d.ts +14 -0
  22. package/dist/src/interfaces/IEmbedder.d.ts +14 -2
  23. package/dist/src/interfaces/ISearcher.d.ts +1 -0
  24. package/dist/src/interfaces/index.d.ts +2 -1
  25. package/dist/src/owner-path.d.ts +10 -0
  26. package/dist/src/owner-path.js +27 -0
  27. package/dist/src/services/AstChunker.d.ts +31 -0
  28. package/dist/src/services/AstChunker.js +114 -0
  29. package/dist/src/services/AstParserService.d.ts +6 -0
  30. package/dist/src/services/AstParserService.js +31 -2
  31. package/dist/src/services/DatabaseService.d.ts +13 -1
  32. package/dist/src/services/DatabaseService.js +70 -25
  33. package/dist/src/services/EmbedderFactory.d.ts +9 -0
  34. package/dist/src/services/EmbedderFactory.js +29 -0
  35. package/dist/src/services/IndexerService.d.ts +17 -5
  36. package/dist/src/services/IndexerService.js +101 -38
  37. package/dist/src/services/InfinityEmbedder.d.ts +79 -0
  38. package/dist/src/services/InfinityEmbedder.js +199 -0
  39. package/dist/src/services/OllamaEmbedder.d.ts +16 -5
  40. package/dist/src/services/OllamaEmbedder.js +84 -36
  41. package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
  42. package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
  43. package/dist/src/services/SearcherService.d.ts +7 -1
  44. package/dist/src/services/SearcherService.js +35 -16
  45. package/dist/src/services/VoyageEmbedder.d.ts +22 -0
  46. package/dist/src/services/VoyageEmbedder.js +72 -0
  47. package/dist/src/services/WebServerService.d.ts +16 -11
  48. package/dist/src/services/WebServerService.js +44 -24
  49. package/dist/src/services/http-retry.d.ts +19 -0
  50. package/dist/src/services/http-retry.js +54 -0
  51. package/dist/src/services/reranker.d.ts +15 -0
  52. package/dist/src/services/reranker.js +44 -0
  53. package/dist/src/setup.d.ts +6 -0
  54. package/dist/src/setup.js +243 -25
  55. package/dist/src/tools/SearchCodebaseTool.js +1 -1
  56. package/dist/src/tools/format-helpers.js +2 -1
  57. package/dist/src/web/dashboard.js +5 -0
  58. package/docker/docker-compose.infinity-gpu.yml +19 -0
  59. package/docker/docker-compose.yml +25 -0
  60. package/package.json +21 -2
  61. package/dist/src/ast.d.ts +0 -2
  62. package/dist/src/ast.js +0 -110
  63. package/dist/src/commands/SetupCommand.d.ts +0 -7
  64. package/dist/src/commands/SetupCommand.js +0 -14
  65. package/dist/src/db/data-source.d.ts +0 -2
  66. package/dist/src/db/data-source.js +0 -13
  67. package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
  68. package/dist/src/db/entities/code-chunk.entity.js +0 -54
  69. package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
  70. package/dist/src/db/entities/code-dependency.entity.js +0 -54
  71. package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
  72. package/dist/src/db/entities/code-symbol.entity.js +0 -59
  73. package/dist/src/db/init.d.ts +0 -2
  74. package/dist/src/db/init.js +0 -62
  75. package/dist/src/env-loader.d.ts +0 -1
  76. package/dist/src/env-loader.js +0 -41
  77. package/dist/src/indexer.d.ts +0 -17
  78. package/dist/src/indexer.js +0 -242
  79. package/dist/src/mcp.d.ts +0 -1
  80. package/dist/src/mcp.js +0 -403
  81. package/dist/src/ollama.d.ts +0 -2
  82. package/dist/src/ollama.js +0 -6
  83. package/dist/src/searcher.d.ts +0 -74
  84. package/dist/src/searcher.js +0 -446
  85. package/dist/vitest.config.d.ts +0 -2
  86. package/dist/vitest.config.js +0 -9
@@ -0,0 +1,25 @@
1
+ import type { Config } from '../config';
2
+ import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
3
+ import type { ILogger } from '../interfaces/ILogger';
4
+ export interface EmbedderDeps {
5
+ fetchImpl?: typeof fetch;
6
+ baseDelayMs?: number;
7
+ }
8
+ /**
9
+ * Embedder for any server exposing an OpenAI-style `/embeddings` endpoint:
10
+ * OpenAI cloud, Infinity, TEI, vLLM, LM Studio, LocalAI. One request carries all
11
+ * inputs (`{ input: string[], model, dimensions? }`) and returns
12
+ * `{ data: [{ embedding, index }] }`, re-ordered to input order.
13
+ */
14
+ export declare class OpenAICompatibleEmbedder implements IEmbedder {
15
+ protected config: Config;
16
+ private logger;
17
+ readonly dimensions: number;
18
+ rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
19
+ private readonly url;
20
+ private readonly doFetch;
21
+ private readonly baseDelayMs?;
22
+ constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
23
+ embed(text: string, inputType?: InputType): Promise<number[]>;
24
+ embedBatch(texts: string[], _inputType?: InputType): Promise<number[][]>;
25
+ }
@@ -0,0 +1,75 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.OpenAICompatibleEmbedder = void 0;
4
+ const LoggerService_1 = require("./LoggerService");
5
+ const config_1 = require("../config");
6
+ const reranker_1 = require("./reranker");
7
+ const http_retry_1 = require("./http-retry");
8
+ const constants_1 = require("../constants");
9
+ /**
10
+ * Embedder for any server exposing an OpenAI-style `/embeddings` endpoint:
11
+ * OpenAI cloud, Infinity, TEI, vLLM, LM Studio, LocalAI. One request carries all
12
+ * inputs (`{ input: string[], model, dimensions? }`) and returns
13
+ * `{ data: [{ embedding, index }] }`, re-ordered to input order.
14
+ */
15
+ class OpenAICompatibleEmbedder {
16
+ config;
17
+ logger;
18
+ dimensions;
19
+ rerank;
20
+ url;
21
+ doFetch;
22
+ baseDelayMs;
23
+ constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
24
+ this.config = config;
25
+ this.logger = logger;
26
+ const base = config.embedding.baseURL.replace(/\/+$/, '');
27
+ // Allow a baseURL that already includes the path (…/v1) or not.
28
+ this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
29
+ this.doFetch = deps.fetchImpl ?? fetch;
30
+ this.baseDelayMs = deps.baseDelayMs;
31
+ this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
32
+ this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
33
+ }
34
+ async embed(text, inputType) {
35
+ const [v] = await this.embedBatch([text], inputType);
36
+ return v;
37
+ }
38
+ async embedBatch(texts, _inputType) {
39
+ if (texts.length === 0)
40
+ return [];
41
+ return (0, http_retry_1.withRetry)(async () => {
42
+ const headers = { 'Content-Type': 'application/json' };
43
+ // Resolve the key lazily — an unset `env:` ref throws here (request time),
44
+ // not at construction, so `rag-web`/`rag-mcp` still start.
45
+ const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
46
+ if (apiKey)
47
+ headers['Authorization'] = `Bearer ${apiKey}`;
48
+ const body = { model: this.config.embedding.model, input: texts };
49
+ if (this.config.embedding.dimensions != null)
50
+ body.dimensions = this.config.embedding.dimensions;
51
+ const res = await this.doFetch(this.url, {
52
+ method: 'POST',
53
+ headers,
54
+ signal: AbortSignal.timeout(120_000),
55
+ body: JSON.stringify(body),
56
+ });
57
+ if (!res.ok) {
58
+ const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
59
+ throw new http_retry_1.HttpError(res.status, `embeddings request failed: ${res.status} ${errBody}`);
60
+ }
61
+ const data = await res.json();
62
+ // Guard against a partial/mismatched response — a missing vector would
63
+ // otherwise be inserted as "[undefined]" and fail the whole file, or (with
64
+ // reordering + no index) silently misalign vectors to the wrong chunks.
65
+ if (!Array.isArray(data.data) || data.data.length !== texts.length) {
66
+ throw new Error(`embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
67
+ }
68
+ // Sort by `index` so vectors line up with input order regardless of server ordering.
69
+ return [...data.data]
70
+ .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
71
+ .map(d => d.embedding);
72
+ }, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'openai-embed' });
73
+ }
74
+ }
75
+ exports.OpenAICompatibleEmbedder = OpenAICompatibleEmbedder;
@@ -9,6 +9,7 @@ export interface ChunkRow {
9
9
  filePath: string;
10
10
  startLine: number;
11
11
  content: string;
12
+ ownerPath?: string | null;
12
13
  }
13
14
  export interface SearchResult extends ChunkRow {
14
15
  score: number;
@@ -102,6 +103,12 @@ export declare class SearcherService {
102
103
  query: string;
103
104
  translated: boolean;
104
105
  }>;
106
+ /**
107
+ * Refuse when the configured embedding model disagrees with the one that built
108
+ * the index — embedding a query into a foreign vector space returns noise. A
109
+ * missing `index_meta` (un-migrated DB / not yet indexed) does not block.
110
+ */
111
+ assertIndexMatches(): Promise<void>;
105
112
  listProjects(): Promise<ProjectRow[]>;
106
113
  searchChunks(query: string, limit: number, project: string | null, threshold?: number): Promise<SearchResult[]>;
107
114
  fetchNeighbors(results: SearchResult[]): Promise<NeighborResult[]>;
@@ -125,7 +132,6 @@ export declare class SearcherService {
125
132
  };
126
133
  readFileAbsolute(absPath: string, maxLines: number): Promise<FileRangeResult>;
127
134
  readFileRange(filePath: string, startLine: number, endLine: number): Promise<FileRangeResult>;
128
- private isNonEnglish;
129
135
  private walkDir;
130
136
  private buildFindSymbolPattern;
131
137
  private inferKind;
@@ -127,16 +127,37 @@ class SearcherService {
127
127
  }
128
128
  return edges;
129
129
  }
130
+ // Query translation was removed (legacy). The query is embedded as-is; the
131
+ // method is kept so the tool layer needs no change and `translated` is always
132
+ // false.
130
133
  async prepareQuery(raw) {
131
- const t0 = Date.now();
132
- if (this.config.search.autoTranslate && this.isNonEnglish(raw) && this.embedder.translate) {
133
- const query = await this.embedder.translate(raw, this.config.search.translateModel);
134
- this.logger.debug('SearcherService: prepareQuery', { translated: true, raw: raw.slice(0, 120), query: query.slice(0, 120), ms: Date.now() - t0 });
135
- return { query, translated: true };
136
- }
137
- this.logger.debug('SearcherService: prepareQuery', { translated: false, query: raw.slice(0, 120), ms: Date.now() - t0 });
138
134
  return { query: raw, translated: false };
139
135
  }
136
+ /**
137
+ * Refuse when the configured embedding model disagrees with the one that built
138
+ * the index — embedding a query into a foreign vector space returns noise. A
139
+ * missing `index_meta` (un-migrated DB / not yet indexed) does not block.
140
+ */
141
+ async assertIndexMatches() {
142
+ const meta = await this.db.readIndexMeta();
143
+ if (!meta)
144
+ return;
145
+ const { provider, model } = this.config.embedding;
146
+ if (meta.provider !== provider || meta.model !== model) {
147
+ throw new Error(`Index was built with ${meta.provider}/${meta.model} but the config selects ${provider}/${model}. ` +
148
+ `Run \`rag-index --reindex\` to rebuild the index for the new model.`);
149
+ }
150
+ // Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
151
+ // override) would embed the query at a width the column can't hold — pgvector
152
+ // would throw an opaque "different vector dimensions" error mid-query. Catch
153
+ // it here with the friendly hint. `embedder.dimensions` is 0 when the width is
154
+ // only knowable by probing, in which case we skip (can't compare cheaply).
155
+ const dim = this.embedder.dimensions;
156
+ if (dim > 0 && dim !== meta.dimensions) {
157
+ throw new Error(`Index was built at ${meta.dimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
158
+ `Run \`rag-index --reindex\` to rebuild the index at the new dimension.`);
159
+ }
160
+ }
140
161
  async listProjects() {
141
162
  const rows = await this.db.query(`SELECT "serviceName", COUNT(*)::int AS chunks
142
163
  FROM code_chunk
@@ -150,21 +171,22 @@ class SearcherService {
150
171
  }
151
172
  async searchChunks(query, limit, project, threshold) {
152
173
  const t0 = Date.now();
174
+ await this.assertIndexMatches();
153
175
  const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
154
176
  const fetchLimit = this.embedder.rerank
155
- ? Math.min(limit * this.config.search.rerankerOverfetch, 200)
177
+ ? Math.min(limit * this.config.reranker.overfetch, 200)
156
178
  : limit;
157
179
  this.logger.debug('SearcherService: searchChunks', {
158
180
  query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
159
181
  });
160
- const embedding = await this.embedder.embed(query);
182
+ const embedding = await this.embedder.embed(query, 'query');
161
183
  const params = [`[${embedding}]`, effectiveThreshold, fetchLimit];
162
184
  let projectFilter = '';
163
185
  if (project) {
164
186
  params.push(project);
165
187
  projectFilter = `AND "serviceName" = $${params.length}`;
166
188
  }
167
- const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content,
189
+ const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
168
190
  ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
169
191
  FROM code_chunk
170
192
  WHERE embedding <=> $1 < $2
@@ -182,7 +204,9 @@ class SearcherService {
182
204
  try {
183
205
  const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
184
206
  this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
185
- return indices.filter(i => i >= 0 && i < candidates.length).map(i => candidates[i]);
207
+ // Cap at `limit`: some rerank servers ignore `top_n` and echo back every
208
+ // candidate, so slicing here keeps the result count within the request.
209
+ return indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
186
210
  }
187
211
  catch (err) {
188
212
  this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
@@ -602,11 +626,6 @@ class SearcherService {
602
626
  this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
603
627
  return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
604
628
  }
605
- isNonEnglish(text) {
606
- // Use Unicode script detection: check for CJK, Cyrillic, Arabic, Hebrew, Thai, etc.
607
- // Excludes common non-ASCII punctuation (smart quotes, em-dashes, accented Latin chars)
608
- return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Cyrillic}\p{Script=Arabic}\p{Script=Hebrew}\p{Script=Thai}\p{Script=Devanagari}]/u.test(text);
609
- }
610
629
  walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
611
630
  let entries;
612
631
  try {
@@ -0,0 +1,22 @@
1
+ import type { Config } from '../config';
2
+ import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
3
+ import type { ILogger } from '../interfaces/ILogger';
4
+ import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
5
+ /**
6
+ * Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
7
+ * of strings, `input_type` (`query` | `document`) is required for best quality,
8
+ * and the optional `output_dimension` requests Matryoshka truncation. Response
9
+ * is `{ data: [{ embedding, index }] }`.
10
+ */
11
+ export declare class VoyageEmbedder implements IEmbedder {
12
+ private config;
13
+ private logger;
14
+ readonly dimensions: number;
15
+ rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
16
+ private readonly url;
17
+ private readonly doFetch;
18
+ private readonly baseDelayMs?;
19
+ constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
20
+ embed(text: string, inputType?: InputType): Promise<number[]>;
21
+ embedBatch(texts: string[], inputType?: InputType): Promise<number[][]>;
22
+ }
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.VoyageEmbedder = void 0;
4
+ const LoggerService_1 = require("./LoggerService");
5
+ const config_1 = require("../config");
6
+ const reranker_1 = require("./reranker");
7
+ const http_retry_1 = require("./http-retry");
8
+ const constants_1 = require("../constants");
9
+ /**
10
+ * Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
11
+ * of strings, `input_type` (`query` | `document`) is required for best quality,
12
+ * and the optional `output_dimension` requests Matryoshka truncation. Response
13
+ * is `{ data: [{ embedding, index }] }`.
14
+ */
15
+ class VoyageEmbedder {
16
+ config;
17
+ logger;
18
+ dimensions;
19
+ rerank;
20
+ url;
21
+ doFetch;
22
+ baseDelayMs;
23
+ constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
24
+ this.config = config;
25
+ this.logger = logger;
26
+ const base = config.embedding.baseURL.replace(/\/+$/, '');
27
+ this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
28
+ this.doFetch = deps.fetchImpl ?? fetch;
29
+ this.baseDelayMs = deps.baseDelayMs;
30
+ this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
31
+ this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
32
+ }
33
+ async embed(text, inputType) {
34
+ const [v] = await this.embedBatch([text], inputType);
35
+ return v;
36
+ }
37
+ async embedBatch(texts, inputType = 'document') {
38
+ if (texts.length === 0)
39
+ return [];
40
+ return (0, http_retry_1.withRetry)(async () => {
41
+ const headers = { 'Content-Type': 'application/json' };
42
+ const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
43
+ if (apiKey)
44
+ headers['Authorization'] = `Bearer ${apiKey}`;
45
+ const body = {
46
+ model: this.config.embedding.model,
47
+ input: texts,
48
+ input_type: inputType,
49
+ };
50
+ if (this.config.embedding.dimensions != null)
51
+ body.output_dimension = this.config.embedding.dimensions;
52
+ const res = await this.doFetch(this.url, {
53
+ method: 'POST',
54
+ headers,
55
+ signal: AbortSignal.timeout(120_000),
56
+ body: JSON.stringify(body),
57
+ });
58
+ if (!res.ok) {
59
+ const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
60
+ throw new http_retry_1.HttpError(res.status, `Voyage embeddings request failed: ${res.status} ${errBody}`);
61
+ }
62
+ const data = await res.json();
63
+ if (!Array.isArray(data.data) || data.data.length !== texts.length) {
64
+ throw new Error(`Voyage embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
65
+ }
66
+ return [...data.data]
67
+ .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
68
+ .map(d => d.embedding);
69
+ }, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'voyage-embed' });
70
+ }
71
+ }
72
+ exports.VoyageEmbedder = VoyageEmbedder;
@@ -13,19 +13,22 @@ export interface FieldOption {
13
13
  values: string[];
14
14
  strict: boolean;
15
15
  }
16
- /** Placeholder shown to the browser instead of the real DB password. */
16
+ /** Placeholder shown to the browser instead of a real secret. */
17
17
  export declare const PASSWORD_MASK = "********";
18
18
  /**
19
- * Merge a config submitted from the browser with the stored one. If the
20
- * password field still holds the mask (i.e. the user did not change it),
21
- * preserve the previously stored password rather than overwriting it with the
22
- * placeholder. Pure — no I/O. Exported for unit testing.
19
+ * Merge a config submitted from the browser with the stored one. Any secret
20
+ * field (DB password, embedding/reranker API keys) still holding the mask means
21
+ * the user did not change it — restore the stored value rather than overwrite it
22
+ * with the placeholder. `env:` references are not masked and round-trip as-is.
23
+ * Pure — no I/O. Exported for unit testing.
23
24
  */
24
25
  export declare function mergeConfigForSave(incoming: JsonConfig, existing: JsonConfig | null): JsonConfig;
25
26
  /**
26
- * Return a copy of the config with the DB password replaced by the mask, so the
27
- * real secret never reaches the browser. The masked value round-trips through
28
- * `mergeConfigForSave` on save. Pure — no I/O. Exported for unit testing.
27
+ * Return a copy of the config with raw secrets (DB password, raw API keys)
28
+ * replaced by the mask, so real secrets never reach the browser. `env:`
29
+ * references are left verbatim — they are not secret and the dashboard shows
30
+ * them as "set via <VAR>". The masked value round-trips through
31
+ * `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
29
32
  */
30
33
  export declare function maskConfig(cfg: JsonConfig): JsonConfig;
31
34
  /**
@@ -49,9 +52,11 @@ export declare class WebServer {
49
52
  private maskedConfig;
50
53
  private saveConfig;
51
54
  /**
52
- * Choice lists for model fields. The embedding model is strict (only
53
- * dimension-compatible models); the translate/reranker models are suggested
54
- * from the user's installed Ollama models but stay free-text.
55
+ * Choice lists for model fields. Both are non-strict comboboxes (free text
56
+ * allowed): the embedding model suggests the known registry models plus any
57
+ * locally-installed models; the reranker model suggests installed models.
58
+ * Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
59
+ * still work — their dimension is probed at index time.
55
60
  */
56
61
  private fieldOptions;
57
62
  private installedModels;
@@ -42,35 +42,53 @@ const config_store_1 = require("../config-store");
42
42
  const constants_1 = require("../constants");
43
43
  const graph_1 = require("../graph");
44
44
  const dashboard_1 = require("../web/dashboard");
45
- /** Placeholder shown to the browser instead of the real DB password. */
45
+ /** Placeholder shown to the browser instead of a real secret. */
46
46
  exports.PASSWORD_MASK = '********';
47
47
  const MAX_BODY = 1_000_000; // 1 MB — config payloads are tiny
48
+ /** A secret worth masking: a non-empty string that is not an `env:` reference. */
49
+ function isRawSecret(v) {
50
+ return typeof v === 'string' && v !== '' && !v.startsWith('env:');
51
+ }
48
52
  /**
49
- * Merge a config submitted from the browser with the stored one. If the
50
- * password field still holds the mask (i.e. the user did not change it),
51
- * preserve the previously stored password rather than overwriting it with the
52
- * placeholder. Pure — no I/O. Exported for unit testing.
53
+ * Merge a config submitted from the browser with the stored one. Any secret
54
+ * field (DB password, embedding/reranker API keys) still holding the mask means
55
+ * the user did not change it — restore the stored value rather than overwrite it
56
+ * with the placeholder. `env:` references are not masked and round-trip as-is.
57
+ * Pure — no I/O. Exported for unit testing.
53
58
  */
54
59
  function mergeConfigForSave(incoming, existing) {
55
60
  const merged = JSON.parse(JSON.stringify(incoming ?? {}));
56
- if (merged.db && merged.db.password === exports.PASSWORD_MASK) {
57
- const prev = existing?.db?.password;
58
- if (prev !== undefined)
59
- merged.db.password = prev;
60
- else
61
- delete merged.db.password;
62
- }
61
+ const restore = (grp, field) => {
62
+ const g = merged[grp];
63
+ if (g && g[field] === exports.PASSWORD_MASK) {
64
+ const prev = existing?.[grp]?.[field];
65
+ if (prev !== undefined)
66
+ g[field] = prev;
67
+ else
68
+ delete g[field];
69
+ }
70
+ };
71
+ restore('db', 'password');
72
+ restore('embedding', 'apiKey');
73
+ restore('reranker', 'apiKey');
63
74
  return merged;
64
75
  }
65
76
  /**
66
- * Return a copy of the config with the DB password replaced by the mask, so the
67
- * real secret never reaches the browser. The masked value round-trips through
68
- * `mergeConfigForSave` on save. Pure — no I/O. Exported for unit testing.
77
+ * Return a copy of the config with raw secrets (DB password, raw API keys)
78
+ * replaced by the mask, so real secrets never reach the browser. `env:`
79
+ * references are left verbatim — they are not secret and the dashboard shows
80
+ * them as "set via <VAR>". The masked value round-trips through
81
+ * `mergeConfigForSave`. Pure — no I/O. Exported for unit testing.
69
82
  */
70
83
  function maskConfig(cfg) {
71
- if (!cfg.db?.password)
72
- return cfg;
73
- return { ...cfg, db: { ...cfg.db, password: exports.PASSWORD_MASK } };
84
+ const clone = JSON.parse(JSON.stringify(cfg ?? {}));
85
+ if (clone.db?.password)
86
+ clone.db.password = exports.PASSWORD_MASK;
87
+ if (isRawSecret(clone.embedding?.apiKey))
88
+ clone.embedding.apiKey = exports.PASSWORD_MASK;
89
+ if (isRawSecret(clone.reranker?.apiKey))
90
+ clone.reranker.apiKey = exports.PASSWORD_MASK;
91
+ return clone;
74
92
  }
75
93
  /**
76
94
  * Local HTTP dashboard. Serves a schema-driven config editor, a project
@@ -171,15 +189,17 @@ class WebServer {
171
189
  this.json(res, 200, { ok: true });
172
190
  }
173
191
  /**
174
- * Choice lists for model fields. The embedding model is strict (only
175
- * dimension-compatible models); the translate/reranker models are suggested
176
- * from the user's installed Ollama models but stay free-text.
192
+ * Choice lists for model fields. Both are non-strict comboboxes (free text
193
+ * allowed): the embedding model suggests the known registry models plus any
194
+ * locally-installed models; the reranker model suggests installed models.
195
+ * Backends beyond the registry (e.g. arbitrary HuggingFace models on Infinity)
196
+ * still work — their dimension is probed at index time.
177
197
  */
178
198
  async fieldOptions() {
179
199
  const installed = await this.installedModels();
200
+ const registryModels = Object.keys(constants_1.MODEL_REGISTRY);
180
201
  return {
181
- 'ollama.model': { values: [...constants_1.SUPPORTED_EMBED_MODELS], strict: true },
182
- 'search.translateModel': { values: installed, strict: false },
202
+ 'embedding.model': { values: [...new Set([...registryModels, ...installed])], strict: false },
183
203
  'reranker.model': { values: installed, strict: false },
184
204
  };
185
205
  }
@@ -188,7 +208,7 @@ class WebServer {
188
208
  return (await this.embedder.listModels?.()) ?? [];
189
209
  }
190
210
  catch (err) {
191
- this.logger.warn('WebServer: could not list Ollama models', { reason: err instanceof Error ? err.message : String(err) });
211
+ this.logger.warn('WebServer: could not list models', { reason: err instanceof Error ? err.message : String(err) });
192
212
  return [];
193
213
  }
194
214
  }
@@ -0,0 +1,19 @@
1
+ import type { ILogger } from '../interfaces/ILogger';
2
+ export declare class HttpError extends Error {
3
+ status: number;
4
+ constructor(status: number, message: string);
5
+ }
6
+ export interface RetryOpts {
7
+ /** Total attempts including the first (default 4). */
8
+ attempts?: number;
9
+ /** Base backoff in ms; grows exponentially (default 500). Set 0 in tests. */
10
+ baseDelayMs?: number;
11
+ logger?: ILogger;
12
+ label?: string;
13
+ }
14
+ /**
15
+ * Run `fn` with bounded exponential backoff on HTTP 429/5xx and network errors.
16
+ * Re-throws the last error once attempts are exhausted or the error is not
17
+ * transient.
18
+ */
19
+ export declare function withRetry<T>(fn: () => Promise<T>, opts?: RetryOpts): Promise<T>;
@@ -0,0 +1,54 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.HttpError = void 0;
4
+ exports.withRetry = withRetry;
5
+ class HttpError extends Error {
6
+ status;
7
+ constructor(status, message) {
8
+ super(message);
9
+ this.status = status;
10
+ this.name = 'HttpError';
11
+ }
12
+ }
13
+ exports.HttpError = HttpError;
14
+ function sleep(ms) {
15
+ return ms > 0 ? new Promise(r => setTimeout(r, ms)) : Promise.resolve();
16
+ }
17
+ /** True for transient failures worth retrying (rate limit / server error / network / timeout). */
18
+ function isRetryable(err) {
19
+ if (err instanceof HttpError)
20
+ return err.status === 429 || (err.status >= 500 && err.status <= 599);
21
+ // A request timeout (AbortSignal.timeout) rejects with a DOMException named
22
+ // 'TimeoutError' — the most common transient cloud failure, so retry it.
23
+ if (err != null && typeof err === 'object' && err.name === 'TimeoutError')
24
+ return true;
25
+ // Network-level errors (fetch throws) are transient too.
26
+ return err instanceof TypeError;
27
+ }
28
+ /**
29
+ * Run `fn` with bounded exponential backoff on HTTP 429/5xx and network errors.
30
+ * Re-throws the last error once attempts are exhausted or the error is not
31
+ * transient.
32
+ */
33
+ async function withRetry(fn, opts = {}) {
34
+ const attempts = opts.attempts ?? 4;
35
+ const base = opts.baseDelayMs ?? 500;
36
+ let lastErr;
37
+ for (let attempt = 1; attempt <= attempts; attempt++) {
38
+ try {
39
+ return await fn();
40
+ }
41
+ catch (err) {
42
+ lastErr = err;
43
+ if (attempt >= attempts || !isRetryable(err))
44
+ break;
45
+ const delay = base * Math.pow(2, attempt - 1);
46
+ opts.logger?.warn('http-retry: transient failure, retrying', {
47
+ label: opts.label, attempt, attempts, delayMs: delay,
48
+ reason: err instanceof Error ? err.message : String(err),
49
+ });
50
+ await sleep(delay);
51
+ }
52
+ }
53
+ throw lastErr;
54
+ }
@@ -0,0 +1,15 @@
1
+ import type { Config } from '../config';
2
+ import type { ILogger } from '../interfaces/ILogger';
3
+ /**
4
+ * Build a rerank function from `config.reranker`, or return `undefined` when
5
+ * reranking is disabled / unconfigured. Targets an OpenAI-compatible `/rerank`
6
+ * endpoint returning `{ results: [{ index }] }` (Infinity, TEI, vLLM,
7
+ * Jina/Cohere-shape). Ollama has NO rerank API, so it is not a valid backend
8
+ * here — the embedding provider is independent of the rerank server
9
+ * (`reranker.baseURL`), so Ollama embeddings + an Infinity reranker is fine.
10
+ * Returns original indices into `documents`, best-first.
11
+ */
12
+ export declare function makeReranker(config: Config, logger: ILogger, deps?: {
13
+ fetchImpl?: typeof fetch;
14
+ baseDelayMs?: number;
15
+ }): ((query: string, documents: string[], topN: number) => Promise<number[]>) | undefined;
@@ -0,0 +1,44 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.makeReranker = makeReranker;
4
+ const config_1 = require("../config");
5
+ const http_retry_1 = require("./http-retry");
6
+ /**
7
+ * Build a rerank function from `config.reranker`, or return `undefined` when
8
+ * reranking is disabled / unconfigured. Targets an OpenAI-compatible `/rerank`
9
+ * endpoint returning `{ results: [{ index }] }` (Infinity, TEI, vLLM,
10
+ * Jina/Cohere-shape). Ollama has NO rerank API, so it is not a valid backend
11
+ * here — the embedding provider is independent of the rerank server
12
+ * (`reranker.baseURL`), so Ollama embeddings + an Infinity reranker is fine.
13
+ * Returns original indices into `documents`, best-first.
14
+ */
15
+ function makeReranker(config, logger, deps = {}) {
16
+ const r = config.reranker;
17
+ if (!r.enabled)
18
+ return undefined;
19
+ if (!r.model) {
20
+ logger.warn('reranker: enabled=true but reranker.model is not set — reranking disabled');
21
+ return undefined;
22
+ }
23
+ const doFetch = deps.fetchImpl ?? fetch;
24
+ return async (query, documents, topN) => {
25
+ return (0, http_retry_1.withRetry)(async () => {
26
+ const headers = { 'Content-Type': 'application/json' };
27
+ const apiKey = (0, config_1.resolveEnvRef)(r.apiKey);
28
+ if (apiKey)
29
+ headers['Authorization'] = `Bearer ${apiKey}`;
30
+ const res = await doFetch(r.url, {
31
+ method: 'POST',
32
+ headers,
33
+ signal: AbortSignal.timeout(30_000),
34
+ body: JSON.stringify({ model: r.model, query, documents, top_n: topN, truncate: true }),
35
+ });
36
+ if (!res.ok) {
37
+ const body = (await res.text().catch(() => res.statusText)).slice(0, 200);
38
+ throw new http_retry_1.HttpError(res.status, `rerank failed: ${res.status} ${body}`);
39
+ }
40
+ const data = await res.json();
41
+ return data.results.map(x => x.index);
42
+ }, { logger, baseDelayMs: deps.baseDelayMs, label: 'rerank' });
43
+ };
44
+ }
@@ -1,2 +1,8 @@
1
1
  import type { LoggerService } from './services/LoggerService';
2
+ /**
3
+ * Default answer for the "start Docker containers?" prompt: no only when both
4
+ * the database and the embedding backend are already reachable. Pure — exported
5
+ * for unit testing.
6
+ */
7
+ export declare function dockerPromptDefault(dbReachable: boolean, embReachable: boolean): 'y' | 'n';
2
8
  export declare function runSetup(logger?: LoggerService): Promise<void>;