codesearch-mcp 0.2.2-alpha → 0.3.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +60 -33
- package/dist/app.js +46 -22
- package/dist/src/chunking/AstChunker.d.ts +12 -0
- package/dist/src/chunking/AstChunker.js +30 -0
- package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
- package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
- package/dist/src/chunking/FragmentMerger.d.ts +11 -0
- package/dist/src/chunking/FragmentMerger.js +61 -0
- package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
- package/dist/src/chunking/ImportOnlyFilter.js +14 -0
- package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
- package/dist/src/chunking/LangchainTextSplitter.js +133 -0
- package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
- package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
- package/dist/src/chunking/RunPacker.d.ts +12 -0
- package/dist/src/chunking/RunPacker.js +84 -0
- package/dist/src/chunking/create-chunker.d.ts +3 -0
- package/dist/src/chunking/create-chunker.js +15 -0
- package/dist/src/chunking/interfaces.d.ts +27 -0
- package/dist/src/chunking/interfaces.js +2 -0
- package/dist/src/commands/ConfigCommand.js +1 -9
- package/dist/src/commands/EvalCommand.d.ts +11 -0
- package/dist/src/commands/EvalCommand.js +264 -0
- package/dist/src/commands/IndexCommand.js +70 -66
- package/dist/src/commands/McpCommand.js +2 -8
- package/dist/src/commands/WebCommand.js +2 -5
- package/dist/src/config-error.d.ts +0 -4
- package/dist/src/config-error.js +0 -4
- package/dist/src/config-store.d.ts +1 -54
- package/dist/src/config-store.js +8 -69
- package/dist/src/config.d.ts +8 -17
- package/dist/src/config.js +35 -34
- package/dist/src/constants.d.ts +2 -28
- package/dist/src/constants.js +6 -28
- package/dist/src/container.d.ts +6 -0
- package/dist/src/container.js +12 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +1 -2
- package/dist/src/db/schema.d.ts +22 -29
- package/dist/src/db/schema.js +3 -14
- package/dist/src/declarations/DeclarationTree.d.ts +19 -0
- package/dist/src/declarations/DeclarationTree.js +66 -0
- package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
- package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
- package/dist/src/declarations/SourceLines.d.ts +37 -0
- package/dist/src/declarations/SourceLines.js +415 -0
- package/dist/src/dependency-traversal.d.ts +10 -0
- package/dist/src/dependency-traversal.js +39 -0
- package/dist/src/enriched-prompt.d.ts +0 -5
- package/dist/src/enriched-prompt.js +0 -5
- package/dist/src/error-record.d.ts +12 -0
- package/dist/src/error-record.js +248 -0
- package/dist/src/eval-corpus.d.ts +16 -0
- package/dist/src/eval-corpus.js +169 -0
- package/dist/src/eval-results.d.ts +15 -0
- package/dist/src/eval-results.js +397 -0
- package/dist/src/file-utils.d.ts +10 -0
- package/dist/src/file-utils.js +19 -1
- package/dist/src/graph.d.ts +0 -25
- package/dist/src/graph.js +0 -28
- package/dist/src/help.js +4 -1
- package/dist/src/hnsw-settings.d.ts +3 -0
- package/dist/src/hnsw-settings.js +24 -0
- package/dist/src/interfaces/IAstParser.d.ts +5 -30
- package/dist/src/interfaces/IChunker.d.ts +3 -12
- package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
- package/dist/src/interfaces/ICorpusInspector.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +2 -36
- package/dist/src/interfaces/IEmbedder.d.ts +0 -9
- package/dist/src/interfaces/ILogger.d.ts +1 -11
- package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
- package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
- package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +216 -0
- package/dist/src/interfaces/IRetrievalEvaluator.js +2 -0
- package/dist/src/interfaces/ISearcher.d.ts +26 -20
- package/dist/src/interfaces/index.d.ts +1 -0
- package/dist/src/interrupt.js +0 -2
- package/dist/src/owner-path.d.ts +9 -10
- package/dist/src/owner-path.js +11 -23
- package/dist/src/project-scope.d.ts +7 -0
- package/dist/src/project-scope.js +37 -0
- package/dist/src/project-selector.d.ts +1 -0
- package/dist/src/project-selector.js +13 -6
- package/dist/src/retrieval-eval.d.ts +24 -0
- package/dist/src/retrieval-eval.js +500 -0
- package/dist/src/search/result-shaping.d.ts +4 -0
- package/dist/src/search/result-shaping.js +19 -0
- package/dist/src/services/AstParserService.d.ts +28 -23
- package/dist/src/services/AstParserService.js +391 -98
- package/dist/src/services/DatabaseService.d.ts +2 -19
- package/dist/src/services/DatabaseService.js +44 -81
- package/dist/src/services/EmbedderFactory.d.ts +0 -4
- package/dist/src/services/EmbedderFactory.js +0 -5
- package/dist/src/services/IndexerService.d.ts +2 -10
- package/dist/src/services/IndexerService.js +29 -45
- package/dist/src/services/InfinityEmbedder.d.ts +0 -21
- package/dist/src/services/InfinityEmbedder.js +0 -15
- package/dist/src/services/LoggerService.d.ts +7 -5
- package/dist/src/services/LoggerService.js +37 -41
- package/dist/src/services/McpServer.d.ts +25 -10
- package/dist/src/services/McpServer.js +101 -64
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
- package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
- package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
- package/dist/src/services/NxMonorepoResolver.js +5 -56
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
- package/dist/src/services/OpenAICompatibleEmbedder.js +1 -14
- package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
- package/dist/src/services/ProjectDiscoveryService.js +0 -4
- package/dist/src/services/SearcherService.d.ts +127 -70
- package/dist/src/services/SearcherService.js +526 -305
- package/dist/src/services/VoyageEmbedder.d.ts +0 -6
- package/dist/src/services/VoyageEmbedder.js +0 -6
- package/dist/src/services/WebServerService.d.ts +1 -53
- package/dist/src/services/WebServerService.js +11 -57
- package/dist/src/services/http-retry.d.ts +0 -7
- package/dist/src/services/http-retry.js +1 -10
- package/dist/src/services/infinity-provisioning.d.ts +0 -74
- package/dist/src/services/infinity-provisioning.js +5 -88
- package/dist/src/services/legacy-ollama-index.d.ts +0 -16
- package/dist/src/services/legacy-ollama-index.js +0 -23
- package/dist/src/services/reranker.d.ts +0 -9
- package/dist/src/services/reranker.js +46 -12
- package/dist/src/setup.d.ts +11 -23
- package/dist/src/setup.js +98 -71
- package/dist/src/symbol-name.d.ts +0 -12
- package/dist/src/symbol-name.js +0 -13
- package/dist/src/symbol-rows.d.ts +2 -11
- package/dist/src/symbol-rows.js +5 -13
- package/dist/src/tool-call-log.d.ts +2 -0
- package/dist/src/tool-call-log.js +24 -0
- package/dist/src/tools/FindSymbolTool.js +14 -5
- package/dist/src/tools/GetCallersTool.d.ts +3 -1
- package/dist/src/tools/GetCallersTool.js +16 -6
- package/dist/src/tools/GetDependenciesTool.d.ts +3 -1
- package/dist/src/tools/GetDependenciesTool.js +15 -8
- package/dist/src/tools/GetDependentsTool.d.ts +3 -1
- package/dist/src/tools/GetDependentsTool.js +31 -16
- package/dist/src/tools/ListFilesTool.d.ts +3 -1
- package/dist/src/tools/ListFilesTool.js +14 -12
- package/dist/src/tools/ListProjectsTool.js +5 -1
- package/dist/src/tools/ReadFileRangeTool.d.ts +3 -1
- package/dist/src/tools/ReadFileRangeTool.js +12 -11
- package/dist/src/tools/ReadFileTool.d.ts +3 -1
- package/dist/src/tools/ReadFileTool.js +12 -10
- package/dist/src/tools/SearchCodebaseTool.js +7 -2
- package/dist/src/tools/format-helpers.d.ts +10 -1
- package/dist/src/tools/format-helpers.js +145 -8
- package/dist/src/web/dashboard.d.ts +0 -8
- package/dist/src/web/dashboard.js +7 -10
- package/package.json +10 -9
- package/dist/src/services/AstChunker.d.ts +0 -42
- package/dist/src/services/AstChunker.js +0 -158
- package/docs/web-dashboard.md +0 -206
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.EvalCommand = void 0;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const crypto = __importStar(require("crypto"));
|
|
40
|
+
const child_process_1 = require("child_process");
|
|
41
|
+
const retrieval_eval_1 = require("../retrieval-eval");
|
|
42
|
+
const eval_results_1 = require("../eval-results");
|
|
43
|
+
const eval_corpus_1 = require("../eval-corpus");
|
|
44
|
+
const DEFAULT_SUITE_DIR = path.join('benchmarks', 'eval');
|
|
45
|
+
const RESULTS_DIR_NAME = path.join('benchmarks', 'results');
|
|
46
|
+
const KEPT_CORPUS_DIR = path.join('benchmarks', 'corpus');
|
|
47
|
+
const _projectRoot = fs.existsSync(path.join(__dirname, '..', '..', 'package.json'))
|
|
48
|
+
? path.join(__dirname, '..', '..')
|
|
49
|
+
: path.join(__dirname, '..', '..', '..');
|
|
50
|
+
function gitCommit() {
|
|
51
|
+
try {
|
|
52
|
+
return (0, child_process_1.execFileSync)('git', ['rev-parse', 'HEAD'], { cwd: _projectRoot, encoding: 'utf-8' }).trim();
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return 'unknown';
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function hashContent(text) {
|
|
59
|
+
return crypto.createHash('sha256').update(text).digest('hex').slice(0, 16);
|
|
60
|
+
}
|
|
61
|
+
function parseSuiteFlag(args) {
|
|
62
|
+
const i = args.indexOf('--suite');
|
|
63
|
+
if (i === -1)
|
|
64
|
+
return DEFAULT_SUITE_DIR;
|
|
65
|
+
const next = args[i + 1];
|
|
66
|
+
if (next === undefined || next.startsWith('-'))
|
|
67
|
+
throw new Error('--suite requires a path');
|
|
68
|
+
return next;
|
|
69
|
+
}
|
|
70
|
+
function parseBaselineFlag(args) {
|
|
71
|
+
const i = args.indexOf('--baseline');
|
|
72
|
+
if (i === -1)
|
|
73
|
+
return null;
|
|
74
|
+
const next = args[i + 1];
|
|
75
|
+
if (next === undefined || next.startsWith('-'))
|
|
76
|
+
throw new Error('--baseline requires a path');
|
|
77
|
+
return next;
|
|
78
|
+
}
|
|
79
|
+
function parseProjectFlag(args) {
|
|
80
|
+
const i = args.indexOf('--project');
|
|
81
|
+
if (i === -1)
|
|
82
|
+
return null;
|
|
83
|
+
const next = args[i + 1];
|
|
84
|
+
if (next === undefined || next.startsWith('-'))
|
|
85
|
+
throw new Error('--project requires a name');
|
|
86
|
+
return next;
|
|
87
|
+
}
|
|
88
|
+
function loadSuite(suitePath) {
|
|
89
|
+
const resolved = path.resolve(_projectRoot, suitePath);
|
|
90
|
+
const stat = fs.statSync(resolved);
|
|
91
|
+
if (stat.isDirectory()) {
|
|
92
|
+
const files = fs.readdirSync(resolved).filter(f => f.endsWith('.json')).sort();
|
|
93
|
+
const parts = files.map(f => fs.readFileSync(path.join(resolved, f), 'utf-8'));
|
|
94
|
+
const suites = parts.map(text => (0, retrieval_eval_1.parseSuite)(JSON.parse(text)));
|
|
95
|
+
return { suite: (0, retrieval_eval_1.mergeSuites)(suites), corpora: (0, retrieval_eval_1.corporaOf)(suites), rawContent: parts.join('\n'), label: 'all' };
|
|
96
|
+
}
|
|
97
|
+
const text = fs.readFileSync(resolved, 'utf-8');
|
|
98
|
+
const suite = (0, retrieval_eval_1.parseSuite)(JSON.parse(text));
|
|
99
|
+
return { suite, corpora: (0, retrieval_eval_1.corporaOf)([suite]), rawContent: text, label: path.basename(resolved, '.json') };
|
|
100
|
+
}
|
|
101
|
+
class EvalCommand {
|
|
102
|
+
container;
|
|
103
|
+
argv1;
|
|
104
|
+
corpusInspector;
|
|
105
|
+
resultsDir;
|
|
106
|
+
constructor(container, argv1, corpusInspector = new eval_corpus_1.GitCorpusInspector(), resultsDir = path.join(_projectRoot, RESULTS_DIR_NAME)) {
|
|
107
|
+
this.container = container;
|
|
108
|
+
this.argv1 = argv1;
|
|
109
|
+
this.corpusInspector = corpusInspector;
|
|
110
|
+
this.resultsDir = resultsDir;
|
|
111
|
+
}
|
|
112
|
+
async execute() {
|
|
113
|
+
const { db, searcher, config } = this.container;
|
|
114
|
+
const args = process.argv.slice(this.argv1 === 'rag-index' ? 3 : 4);
|
|
115
|
+
let suitePath, baselinePath, projectFilter;
|
|
116
|
+
try {
|
|
117
|
+
suitePath = parseSuiteFlag(args);
|
|
118
|
+
baselinePath = parseBaselineFlag(args);
|
|
119
|
+
projectFilter = parseProjectFlag(args);
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
console.error(`\n${err.message}`);
|
|
123
|
+
process.exitCode = 1;
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
let suite, corpora, rawContent, label;
|
|
127
|
+
try {
|
|
128
|
+
({ suite, corpora, rawContent, label } = loadSuite(suitePath));
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
if (err instanceof retrieval_eval_1.EvalSuiteError) {
|
|
132
|
+
console.error(`\nInvalid suite: ${err.message}`);
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
console.error(`\nFailed to load suite at '${suitePath}': ${err.message}`);
|
|
136
|
+
}
|
|
137
|
+
process.exitCode = 1;
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if (projectFilter) {
|
|
141
|
+
suite = { version: 1, cases: suite.cases.filter(c => c.project === projectFilter) };
|
|
142
|
+
}
|
|
143
|
+
if (suite.cases.length === 0) {
|
|
144
|
+
console.error('\nNo cases to evaluate (suite is empty, or --project matched nothing).');
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
const runCorpora = corpora.filter(c => suite.cases.some(k => k.corpus === c.root));
|
|
149
|
+
let refused = false;
|
|
150
|
+
for (const corpus of runCorpora) {
|
|
151
|
+
let failures;
|
|
152
|
+
try {
|
|
153
|
+
failures = await (0, eval_corpus_1.checkCorpus)(corpus, path.join(config.workspaceRoot, corpus.root), path.join(_projectRoot, KEPT_CORPUS_DIR, corpus.root), this.corpusInspector);
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
failures = [`cannot inspect the checkout: ${err.message}`];
|
|
157
|
+
}
|
|
158
|
+
if (failures.length === 0)
|
|
159
|
+
continue;
|
|
160
|
+
refused = true;
|
|
161
|
+
console.error(`\nCorpus ${corpus.root} does not match ${corpus.commit}:`);
|
|
162
|
+
for (const f of failures)
|
|
163
|
+
console.error(` - ${f}`);
|
|
164
|
+
}
|
|
165
|
+
if (refused) {
|
|
166
|
+
console.error('\nNo search was run and no result file was written.');
|
|
167
|
+
process.exitCode = 1;
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
await db.initDbWithSchema();
|
|
171
|
+
const projectDirs = new Map((await searcher.listProjects()).map(p => [p.serviceName, p.relPath ?? null]));
|
|
172
|
+
const knownFilesCache = new Map();
|
|
173
|
+
const inspect = {
|
|
174
|
+
async isFileIndexed(project, file) {
|
|
175
|
+
if (!knownFilesCache.has(project))
|
|
176
|
+
knownFilesCache.set(project, db.listKnownFiles(project));
|
|
177
|
+
const files = await knownFilesCache.get(project);
|
|
178
|
+
return files.includes(file);
|
|
179
|
+
},
|
|
180
|
+
async ownerPathsOfFile(project, file) {
|
|
181
|
+
const rows = await db.query(`SELECT DISTINCT "ownerPath" FROM code_chunk WHERE "serviceName" = $1 AND "filePath" = $2 AND "ownerPath" IS NOT NULL`, [project, file]);
|
|
182
|
+
return rows.map(r => r.ownerPath);
|
|
183
|
+
},
|
|
184
|
+
async projectDir(project) {
|
|
185
|
+
return projectDirs.get(project) ?? null;
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
const toHits = (rows) => rows.map(r => ({ filePath: r.filePath, ownerPath: r.ownerPath ?? null }));
|
|
189
|
+
const search = async (query, limit, project, threshold) => {
|
|
190
|
+
const prepared = await searcher.prepareQuery(query);
|
|
191
|
+
const r = await searcher.searchChunksWithStages(prepared.query, limit, project, threshold, undefined, limit);
|
|
192
|
+
return {
|
|
193
|
+
pool: toHits(r.pool),
|
|
194
|
+
stages: r.stages.reranked
|
|
195
|
+
? { vector: toHits(r.stages.vector), reranked: toHits(r.stages.reranked) }
|
|
196
|
+
: { vector: toHits(r.stages.vector) },
|
|
197
|
+
rerankFailed: r.rerankFailed,
|
|
198
|
+
fetchLimit: r.fetchLimit,
|
|
199
|
+
indexParams: r.indexParams,
|
|
200
|
+
};
|
|
201
|
+
};
|
|
202
|
+
let outcome;
|
|
203
|
+
try {
|
|
204
|
+
outcome = await new retrieval_eval_1.RetrievalEvaluator().evaluate(suite, search, inspect, p => console.error((0, eval_results_1.renderProgress)(p)));
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
if (!(err instanceof retrieval_eval_1.EvalSuiteError))
|
|
208
|
+
throw err;
|
|
209
|
+
console.error(`\nInvalid suite: ${err.message}`);
|
|
210
|
+
process.exitCode = 1;
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
const { run, observed } = outcome;
|
|
214
|
+
const corpusRecords = [];
|
|
215
|
+
for (const corpus of runCorpora) {
|
|
216
|
+
const projects = [...projectDirs].filter(([, dir]) => dir !== null && (0, retrieval_eval_1.isUnderRoot)(dir, corpus.root)).map(([name]) => name).sort();
|
|
217
|
+
const started = Date.now();
|
|
218
|
+
const record = { ...corpus, projects, ...(await (0, eval_corpus_1.fingerprintCorpus)(db, projects)) };
|
|
219
|
+
console.log(`Corpus ${corpus.root}: ${record.chunks} chunks in ${projects.length} project(s), fingerprint ${record.fingerprint} (${((Date.now() - started) / 1000).toFixed(1)}s)`);
|
|
220
|
+
corpusRecords.push(record);
|
|
221
|
+
}
|
|
222
|
+
const indexMeta = await db.readIndexMeta();
|
|
223
|
+
const metadata = {
|
|
224
|
+
suitePath,
|
|
225
|
+
suiteHash: hashContent(rawContent),
|
|
226
|
+
embeddingProvider: config.embedding.provider,
|
|
227
|
+
embeddingModel: config.embedding.model,
|
|
228
|
+
rerankerModel: config.reranker.enabled ? config.reranker.model : null,
|
|
229
|
+
indexMeta: indexMeta ? { provider: indexMeta.provider, model: indexMeta.model, dimensions: indexMeta.dimensions } : null,
|
|
230
|
+
gitCommit: gitCommit(),
|
|
231
|
+
timestamp: new Date().toISOString(),
|
|
232
|
+
conditions: {
|
|
233
|
+
similarityThreshold: config.search.similarityThreshold,
|
|
234
|
+
overfetch: config.reranker.overfetch,
|
|
235
|
+
candidateFloor: config.reranker.candidateFloor,
|
|
236
|
+
fetchLimit: observed.fetchLimit,
|
|
237
|
+
efSearch: observed.efSearch,
|
|
238
|
+
iterativeScan: observed.iterativeScan,
|
|
239
|
+
},
|
|
240
|
+
corpora: corpusRecords,
|
|
241
|
+
liveCases: suite.cases.filter(c => !c.corpus).length,
|
|
242
|
+
};
|
|
243
|
+
const resultFile = { metadata, run };
|
|
244
|
+
for (const line of (0, eval_results_1.renderSummary)(resultFile))
|
|
245
|
+
console.log(line);
|
|
246
|
+
let comparison;
|
|
247
|
+
if (baselinePath) {
|
|
248
|
+
try {
|
|
249
|
+
const baseline = (0, eval_results_1.readResultFile)(path.resolve(_projectRoot, baselinePath));
|
|
250
|
+
const cmp = (0, retrieval_eval_1.compare)(baseline, resultFile);
|
|
251
|
+
comparison = { baselinePath, result: cmp };
|
|
252
|
+
for (const line of (0, eval_results_1.renderComparison)(baselinePath, cmp))
|
|
253
|
+
console.log(line);
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
console.error(`\nFailed to read baseline '${baselinePath}': ${err.message}`);
|
|
257
|
+
process.exitCode = 1;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const { jsonPath, mdPath } = (0, eval_results_1.writeResultFiles)(this.resultsDir, label, resultFile, new Date(), comparison);
|
|
261
|
+
console.log(`\nResults written to:\n ${path.relative(_projectRoot, jsonPath)}\n ${path.relative(_projectRoot, mdPath)}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
exports.EvalCommand = EvalCommand;
|
|
@@ -5,8 +5,16 @@ const interrupt_1 = require("../interrupt");
|
|
|
5
5
|
const project_selector_1 = require("../project-selector");
|
|
6
6
|
const graph_1 = require("../graph");
|
|
7
7
|
const MonorepoAwareDiscoveryService_1 = require("../services/MonorepoAwareDiscoveryService");
|
|
8
|
+
const project_scope_1 = require("../project-scope");
|
|
8
9
|
const file_utils_1 = require("../file-utils");
|
|
9
10
|
const legacy_ollama_index_1 = require("../services/legacy-ollama-index");
|
|
11
|
+
function parseProjectFlag(args) {
|
|
12
|
+
const i = args.indexOf('--project');
|
|
13
|
+
if (i === -1)
|
|
14
|
+
return { present: false };
|
|
15
|
+
const next = args[i + 1];
|
|
16
|
+
return next !== undefined && !next.startsWith('-') ? { present: true, name: next } : { present: true };
|
|
17
|
+
}
|
|
10
18
|
class IndexCommand {
|
|
11
19
|
container;
|
|
12
20
|
argv1;
|
|
@@ -19,12 +27,41 @@ class IndexCommand {
|
|
|
19
27
|
const args = process.argv.slice(this.argv1 === 'rag-index' ? 2 : 3);
|
|
20
28
|
const force = args.includes('--reindex');
|
|
21
29
|
const forceSelect = args.includes('--select');
|
|
30
|
+
const projectFlag = parseProjectFlag(args);
|
|
31
|
+
let discovered;
|
|
32
|
+
try {
|
|
33
|
+
discovered = discovery.discover(config.workspaceRoot);
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
console.error(`\n${err.message}`);
|
|
37
|
+
process.exitCode = 1;
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
let scanTargets;
|
|
41
|
+
if (projectFlag.present) {
|
|
42
|
+
let name = projectFlag.name;
|
|
43
|
+
if (name === undefined) {
|
|
44
|
+
if (discovered.length === 0) {
|
|
45
|
+
console.error('\nNo projects available to select.');
|
|
46
|
+
process.exitCode = 1;
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
name = await (0, project_selector_1.selectSingleProject)(discovered);
|
|
50
|
+
if (name === undefined) {
|
|
51
|
+
process.exitCode = 1;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const { targets, notFound } = (0, project_scope_1.resolveProjectNames)([name], discovered);
|
|
56
|
+
if (notFound.length > 0) {
|
|
57
|
+
const available = (0, project_scope_1.listAvailableNames)(discovered).join(', ');
|
|
58
|
+
console.error(`\nProject '${notFound[0]}' not found. Available projects: ${available}`);
|
|
59
|
+
process.exitCode = 1;
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
scanTargets = targets;
|
|
63
|
+
}
|
|
22
64
|
await db.initDbWithSchema();
|
|
23
|
-
// A former Ollama installation the config layer cannot recognise (old
|
|
24
|
-
// defaults, no `embedding`/`ollama` block) is protected here: without an
|
|
25
|
-
// explicit `--reindex`, refuse to touch an index built by `ollama` (or a
|
|
26
|
-
// pre-`index_meta` install, which is indistinguishable from one) before
|
|
27
|
-
// any backend readiness check, Docker command, or schema work.
|
|
28
65
|
if (!force) {
|
|
29
66
|
let identity;
|
|
30
67
|
try {
|
|
@@ -43,37 +80,17 @@ class IndexCommand {
|
|
|
43
80
|
return;
|
|
44
81
|
}
|
|
45
82
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
// A project indexed before member extraction mixes old-format rows (no
|
|
54
|
-
// members, bare calleeSymbol) with new ones if left to an incremental run.
|
|
55
|
-
// Refuse before any backend readiness check, Docker command, or schema
|
|
56
|
-
// work — the same pattern as the legacy-Ollama guard above.
|
|
57
|
-
if (!force) {
|
|
58
|
-
let outdated;
|
|
59
|
-
try {
|
|
60
|
-
outdated = await db.listOutdatedProjects(projects.map(p => p.name));
|
|
61
|
-
}
|
|
62
|
-
catch (err) {
|
|
63
|
-
console.error(`\n${err.message}`);
|
|
64
|
-
process.exitCode = 1;
|
|
65
|
-
return;
|
|
66
|
-
}
|
|
67
|
-
if (outdated.length > 0) {
|
|
68
|
-
console.error(`\nIndexed before member indexing: ${[...outdated].sort().join(', ')} — run \`rag-index --reindex\`.`);
|
|
69
|
-
process.exitCode = 1;
|
|
83
|
+
let projects;
|
|
84
|
+
if (scanTargets) {
|
|
85
|
+
projects = scanTargets;
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
const selected = await new project_selector_1.ProjectSelector(config.workspaceRoot).select(discovered, forceSelect);
|
|
89
|
+
if (selected.length === 0)
|
|
70
90
|
return;
|
|
71
|
-
|
|
91
|
+
projects = (0, MonorepoAwareDiscoveryService_1.toLeafProjects)(selected);
|
|
72
92
|
}
|
|
73
|
-
|
|
74
|
-
// resolve the active embedding dimension and reconcile it with index_meta
|
|
75
|
-
// (recreate code_chunk on a dimension change, clear rows on a model change)
|
|
76
|
-
// before scanning. A clear error here beats a mid-run vector width mismatch.
|
|
93
|
+
const fullLeafProjects = (0, MonorepoAwareDiscoveryService_1.toLeafProjects)(discovered);
|
|
77
94
|
try {
|
|
78
95
|
await embedder.ensureModelAvailable?.();
|
|
79
96
|
const meta = await indexer.prepareEmbeddingSchema();
|
|
@@ -87,50 +104,37 @@ class IndexCommand {
|
|
|
87
104
|
const controller = new AbortController();
|
|
88
105
|
const teardown = (0, interrupt_1.setupInterrupt)(controller);
|
|
89
106
|
try {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
107
|
+
if (!scanTargets) {
|
|
108
|
+
let pruneResult;
|
|
109
|
+
try {
|
|
110
|
+
pruneResult = await indexer.pruneProjects(projects.map(p => p.name));
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
console.error(`\nPruning failed: ${err.message}`);
|
|
114
|
+
process.exitCode = 1;
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
const { removed, failed } = pruneResult;
|
|
118
|
+
for (const name of removed) {
|
|
119
|
+
console.log(`Removed project ${name} (not selected)`);
|
|
120
|
+
}
|
|
121
|
+
if (failed.length > 0) {
|
|
122
|
+
process.exitCode = 1;
|
|
123
|
+
}
|
|
105
124
|
}
|
|
106
125
|
const filesPerProject = projects.map(p => (0, file_utils_1.collectFiles)(p.path, (0, file_utils_1.loadIgnore)(p.path), p.path));
|
|
107
126
|
const totalFiles = filesPerProject.reduce((sum, f) => sum + f.length, 0);
|
|
108
127
|
console.log(`\n${projects.length} projects — ${totalFiles} files total`);
|
|
109
128
|
if (force)
|
|
110
129
|
console.log('Reindex mode: all files will be re-indexed.');
|
|
111
|
-
const allStats =
|
|
112
|
-
for (let i = 0; i < projects.length; i++) {
|
|
113
|
-
if (controller.signal.aborted)
|
|
114
|
-
break;
|
|
115
|
-
const stats = await indexer.scanAndIndex(projects[i].path, projects[i].name, force, controller.signal, filesPerProject[i], { rootTsConfigPath: projects[i].rootTsConfigPath ?? undefined });
|
|
116
|
-
allStats.push(stats);
|
|
117
|
-
const remaining = projects.length - allStats.length;
|
|
118
|
-
if (remaining > 0 && !controller.signal.aborted) {
|
|
119
|
-
const leftFiles = filesPerProject.slice(allStats.length).reduce((sum, f) => sum + f.length, 0);
|
|
120
|
-
console.log(` ${allStats.length}/${projects.length} projects — done in ${stats.elapsedSec}s — ${leftFiles} files left`);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
130
|
+
const allStats = await indexer.scanProjects(projects, force, controller.signal, filesPerProject);
|
|
123
131
|
if (controller.signal.aborted)
|
|
124
132
|
console.log('\nIndexing interrupted.');
|
|
125
133
|
this.printSummary(allStats);
|
|
126
134
|
if (!controller.signal.aborted) {
|
|
127
|
-
// The graph is a post-index nicety — a failure here (DB dropped after a
|
|
128
|
-
// long run, unreadable dir) must not fail the whole run: indexing already
|
|
129
|
-
// succeeded and the summary is printed.
|
|
130
135
|
try {
|
|
131
|
-
// Real import edges (code_dependency, just written) → intra-repo graph links.
|
|
132
136
|
const projectEdges = await searcher.getProjectEdges();
|
|
133
|
-
(0, graph_1.generateProjectGraph)(
|
|
137
|
+
(0, graph_1.generateProjectGraph)(fullLeafProjects, projectEdges, config.workspaceRoot);
|
|
134
138
|
}
|
|
135
139
|
catch (err) {
|
|
136
140
|
console.error(` ⚠ Could not build project graph: ${err.message}`);
|
|
@@ -11,25 +11,19 @@ class McpCommand {
|
|
|
11
11
|
let startupError;
|
|
12
12
|
try {
|
|
13
13
|
await db.initDb();
|
|
14
|
-
// Refuse to serve searches embedded into a foreign vector space — surface
|
|
15
|
-
// the mismatch through the existing startup-error plumbing so every tool
|
|
16
|
-
// call reports it clearly.
|
|
17
14
|
await searcher.assertIndexMatches();
|
|
18
15
|
}
|
|
19
16
|
catch (err) {
|
|
20
17
|
startupError = err instanceof Error ? err : new Error(String(err));
|
|
21
|
-
logger.error('McpCommand: startup check failed', {
|
|
18
|
+
logger.error('McpCommand: startup check failed', { err: startupError });
|
|
22
19
|
}
|
|
23
20
|
try {
|
|
24
21
|
await mcp.start(startupError);
|
|
25
22
|
}
|
|
26
23
|
catch (err) {
|
|
27
|
-
|
|
28
|
-
logger.error('McpCommand: MCP server failed to start', { reason });
|
|
24
|
+
logger.error('McpCommand: MCP server failed to start', { err });
|
|
29
25
|
return;
|
|
30
26
|
}
|
|
31
|
-
// server.connect() resolves immediately after setup; keep the process alive
|
|
32
|
-
// until the MCP client disconnects (stdin closes)
|
|
33
27
|
await new Promise(resolve => {
|
|
34
28
|
process.stdin.once('close', resolve);
|
|
35
29
|
process.stdin.once('end', resolve);
|
|
@@ -17,9 +17,8 @@ class WebCommand {
|
|
|
17
17
|
}
|
|
18
18
|
catch (err) {
|
|
19
19
|
const reason = err instanceof Error ? err.message : String(err);
|
|
20
|
-
logger.error('WebCommand: database connection failed', {
|
|
20
|
+
logger.error('WebCommand: database connection failed', { err });
|
|
21
21
|
console.error(`Database connection failed: ${reason}`);
|
|
22
|
-
// Continue: config editing still works; project/graph tabs will error in-page.
|
|
23
22
|
}
|
|
24
23
|
const server = new WebServerService_1.WebServer(searcher, config, logger, embedder);
|
|
25
24
|
const host = '127.0.0.1';
|
|
@@ -28,13 +27,11 @@ class WebCommand {
|
|
|
28
27
|
}
|
|
29
28
|
catch (err) {
|
|
30
29
|
const reason = err instanceof Error ? err.message : String(err);
|
|
31
|
-
logger.error('WebCommand: server failed to start', {
|
|
30
|
+
logger.error('WebCommand: server failed to start', { err });
|
|
32
31
|
console.error(`Failed to start web server on ${host}:${this.port}: ${reason}`);
|
|
33
32
|
return;
|
|
34
33
|
}
|
|
35
34
|
console.log(`RAG dashboard: http://${host}:${this.port} (Ctrl+C to stop)`);
|
|
36
|
-
// Keep execute() pending until a shutdown signal, otherwise app.ts tears
|
|
37
|
-
// down the DB and exits immediately (mirrors McpCommand's keep-alive).
|
|
38
35
|
await new Promise(resolve => {
|
|
39
36
|
const shutdown = () => { void server.stop().then(resolve); };
|
|
40
37
|
process.once('SIGINT', shutdown);
|
package/dist/src/config-error.js
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ConfigError = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* Own module (not `config.ts`) so `config-store.ts` can throw it too without
|
|
6
|
-
* an import cycle — `config.ts` imports from `config-store.ts`.
|
|
7
|
-
*/
|
|
8
4
|
class ConfigError extends Error {
|
|
9
5
|
constructor(message) {
|
|
10
6
|
super(message);
|
|
@@ -42,6 +42,7 @@ declare const jsonSchema: z.ZodObject<{
|
|
|
42
42
|
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
43
43
|
provider: z.ZodOptional<z.ZodEnum<{
|
|
44
44
|
"openai-compatible": "openai-compatible";
|
|
45
|
+
voyage: "voyage";
|
|
45
46
|
}>>;
|
|
46
47
|
baseURL: z.ZodOptional<z.ZodString>;
|
|
47
48
|
apiKey: z.ZodOptional<z.ZodString>;
|
|
@@ -58,24 +59,11 @@ declare const jsonSchema: z.ZodObject<{
|
|
|
58
59
|
warn: "warn";
|
|
59
60
|
}>>;
|
|
60
61
|
dir: z.ZodOptional<z.ZodString>;
|
|
61
|
-
searchMetrics: z.ZodOptional<z.ZodBoolean>;
|
|
62
62
|
}, z.core.$strip>>;
|
|
63
63
|
workspaceRoot: z.ZodOptional<z.ZodString>;
|
|
64
64
|
}, z.core.$strip>;
|
|
65
65
|
export type JsonConfig = z.infer<typeof jsonSchema>;
|
|
66
66
|
export declare const ALL_KEYS: string[];
|
|
67
|
-
/**
|
|
68
|
-
* Single source of truth for built-in defaults, keyed by raw config.json dotted
|
|
69
|
-
* key. Consumed both by the web dashboard (`GET /api/defaults`) and by
|
|
70
|
-
* `resolvedSchema` in `src/config.ts`, whose `.default(...)` calls reference
|
|
71
|
-
* these values — so the runtime default and the dashboard-displayed default
|
|
72
|
-
* cannot diverge. `as const` keeps the literal types so those `.default()`
|
|
73
|
-
* lookups stay type-safe. Keys omitted here have no static default: required
|
|
74
|
-
* fields (db.host/user/password) and values derived at runtime — db.name →
|
|
75
|
-
* rag_<user>, and the provider-dependent URLs (embedding.baseURL,
|
|
76
|
-
* reranker.baseURL, reranker.url), which are computed in the `Config`
|
|
77
|
-
* constructor per provider rather than through a schema `.default(...)`.
|
|
78
|
-
*/
|
|
79
67
|
export declare const CONFIG_DEFAULTS: {
|
|
80
68
|
readonly 'db.port': 5433;
|
|
81
69
|
readonly 'embedding.provider': "infinity";
|
|
@@ -97,54 +85,13 @@ export declare const CONFIG_DEFAULTS: {
|
|
|
97
85
|
readonly 'reranker.overfetch': 5;
|
|
98
86
|
readonly 'reranker.candidateFloor': 80;
|
|
99
87
|
readonly 'logging.level': "error";
|
|
100
|
-
readonly 'logging.searchMetrics': false;
|
|
101
88
|
};
|
|
102
89
|
export declare function readConfigJson(): JsonConfig | null;
|
|
103
|
-
/**
|
|
104
|
-
* Detect a raw (pre-migration) config object that still selects the removed
|
|
105
|
-
* `ollama` embedding provider — explicitly, via a legacy top-level `ollama`
|
|
106
|
-
* block, or implicitly (no `provider` set, but `baseURL`/`model` point at
|
|
107
|
-
* Ollama's defaults) — and throw a `ConfigError` with migration guidance
|
|
108
|
-
* instead of letting it validate under a silently reinterpreted provider.
|
|
109
|
-
* Runs before `migrateRawConfig` so a legacy `ollama.*` block is never
|
|
110
|
-
* converted into an `embedding` block first. Pure — never mutates `raw`.
|
|
111
|
-
*/
|
|
112
90
|
export declare function detectFormerOllamaConfig(raw: unknown): void;
|
|
113
|
-
/**
|
|
114
|
-
* Migrate a legacy raw config object to the current shape, in place of a
|
|
115
|
-
* versioned migration system. Pure — returns a new object, never mutates input.
|
|
116
|
-
*
|
|
117
|
-
* - Removed translation keys (`search.autoTranslate`, `search.translateModel`)
|
|
118
|
-
* are dropped so they neither validate-fail nor take effect.
|
|
119
|
-
* - Legacy `reranker.provider: 'ollama'` (a dead value — Ollama has no rerank
|
|
120
|
-
* API) is rewritten to `openai-compatible`, and a stale `/api/rerank` URL is
|
|
121
|
-
* dropped so it re-derives to `/rerank`. Otherwise the tightened enum would
|
|
122
|
-
* fail validation on existing configs and lock the user out.
|
|
123
|
-
*
|
|
124
|
-
* A legacy top-level `ollama` block reaches here only when `detectFormerOllamaConfig`
|
|
125
|
-
* did not already reject it (i.e. an `embedding` block is also present); it is
|
|
126
|
-
* dropped rather than merged. Exported for unit testing.
|
|
127
|
-
*/
|
|
128
91
|
export declare function migrateRawConfig(raw: unknown): unknown;
|
|
129
|
-
/**
|
|
130
|
-
* Validate a raw (pre-migration) config object against the Zod schema: first
|
|
131
|
-
* `detectFormerOllamaConfig` (a `ConfigError` on a former-Ollama config takes
|
|
132
|
-
* priority over both migration and the regular Zod enum rejection), then
|
|
133
|
-
* `migrateRawConfig`, then the schema itself. Unknown keys (e.g. the "$schema"
|
|
134
|
-
* editor reference) are stripped. Throws with a caller-friendly message on
|
|
135
|
-
* validation failure. The single entry point for both `readConfigJson` and
|
|
136
|
-
* `PUT /api/config`, so both apply identical rules with an identical message.
|
|
137
|
-
* Pure — no file I/O.
|
|
138
|
-
*/
|
|
139
92
|
export declare function parseConfig(raw: unknown, source?: string): JsonConfig;
|
|
140
|
-
/** Build the JSON Schema object generated from the Zod config schema. Pure. */
|
|
141
93
|
export declare function buildJsonSchema(): Record<string, unknown>;
|
|
142
94
|
export declare function writeConfigJson(data: JsonConfig): void;
|
|
143
|
-
/**
|
|
144
|
-
* Generate a JSON Schema from the Zod config schema and write it next to
|
|
145
|
-
* config.json. Editors resolve it via the "$schema" reference and provide
|
|
146
|
-
* key autocomplete, type hints, and inline validation while editing.
|
|
147
|
-
*/
|
|
148
95
|
export declare function writeSchemaFile(): void;
|
|
149
96
|
export declare function getNestedValue(cfg: JsonConfig, key: string): unknown;
|
|
150
97
|
export declare function setNestedValue(cfg: JsonConfig, key: string, raw: string): JsonConfig;
|