codesearch-mcp 0.2.2-alpha → 0.3.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +60 -33
- package/dist/app.js +46 -22
- package/dist/src/chunking/AstChunker.d.ts +12 -0
- package/dist/src/chunking/AstChunker.js +30 -0
- package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
- package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
- package/dist/src/chunking/FragmentMerger.d.ts +11 -0
- package/dist/src/chunking/FragmentMerger.js +61 -0
- package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
- package/dist/src/chunking/ImportOnlyFilter.js +14 -0
- package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
- package/dist/src/chunking/LangchainTextSplitter.js +133 -0
- package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
- package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
- package/dist/src/chunking/RunPacker.d.ts +12 -0
- package/dist/src/chunking/RunPacker.js +84 -0
- package/dist/src/chunking/create-chunker.d.ts +3 -0
- package/dist/src/chunking/create-chunker.js +15 -0
- package/dist/src/chunking/interfaces.d.ts +27 -0
- package/dist/src/chunking/interfaces.js +2 -0
- package/dist/src/commands/ConfigCommand.js +1 -9
- package/dist/src/commands/EvalCommand.d.ts +11 -0
- package/dist/src/commands/EvalCommand.js +264 -0
- package/dist/src/commands/IndexCommand.js +70 -66
- package/dist/src/commands/McpCommand.js +2 -8
- package/dist/src/commands/WebCommand.js +2 -5
- package/dist/src/config-error.d.ts +0 -4
- package/dist/src/config-error.js +0 -4
- package/dist/src/config-store.d.ts +1 -54
- package/dist/src/config-store.js +8 -69
- package/dist/src/config.d.ts +8 -17
- package/dist/src/config.js +35 -34
- package/dist/src/constants.d.ts +2 -28
- package/dist/src/constants.js +6 -28
- package/dist/src/container.d.ts +6 -0
- package/dist/src/container.js +12 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +1 -2
- package/dist/src/db/schema.d.ts +22 -29
- package/dist/src/db/schema.js +3 -14
- package/dist/src/declarations/DeclarationTree.d.ts +19 -0
- package/dist/src/declarations/DeclarationTree.js +66 -0
- package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
- package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
- package/dist/src/declarations/SourceLines.d.ts +37 -0
- package/dist/src/declarations/SourceLines.js +415 -0
- package/dist/src/dependency-traversal.d.ts +10 -0
- package/dist/src/dependency-traversal.js +39 -0
- package/dist/src/enriched-prompt.d.ts +0 -5
- package/dist/src/enriched-prompt.js +0 -5
- package/dist/src/error-record.d.ts +12 -0
- package/dist/src/error-record.js +248 -0
- package/dist/src/eval-corpus.d.ts +16 -0
- package/dist/src/eval-corpus.js +169 -0
- package/dist/src/eval-results.d.ts +15 -0
- package/dist/src/eval-results.js +397 -0
- package/dist/src/file-utils.d.ts +10 -0
- package/dist/src/file-utils.js +19 -1
- package/dist/src/graph.d.ts +0 -25
- package/dist/src/graph.js +0 -28
- package/dist/src/help.js +4 -1
- package/dist/src/hnsw-settings.d.ts +3 -0
- package/dist/src/hnsw-settings.js +24 -0
- package/dist/src/interfaces/IAstParser.d.ts +5 -30
- package/dist/src/interfaces/IChunker.d.ts +3 -12
- package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
- package/dist/src/interfaces/ICorpusInspector.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +2 -36
- package/dist/src/interfaces/IEmbedder.d.ts +0 -9
- package/dist/src/interfaces/ILogger.d.ts +1 -11
- package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
- package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
- package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +216 -0
- package/dist/src/interfaces/IRetrievalEvaluator.js +2 -0
- package/dist/src/interfaces/ISearcher.d.ts +26 -20
- package/dist/src/interfaces/index.d.ts +1 -0
- package/dist/src/interrupt.js +0 -2
- package/dist/src/owner-path.d.ts +9 -10
- package/dist/src/owner-path.js +11 -23
- package/dist/src/project-scope.d.ts +7 -0
- package/dist/src/project-scope.js +37 -0
- package/dist/src/project-selector.d.ts +1 -0
- package/dist/src/project-selector.js +13 -6
- package/dist/src/retrieval-eval.d.ts +24 -0
- package/dist/src/retrieval-eval.js +500 -0
- package/dist/src/search/result-shaping.d.ts +4 -0
- package/dist/src/search/result-shaping.js +19 -0
- package/dist/src/services/AstParserService.d.ts +28 -23
- package/dist/src/services/AstParserService.js +391 -98
- package/dist/src/services/DatabaseService.d.ts +2 -19
- package/dist/src/services/DatabaseService.js +44 -81
- package/dist/src/services/EmbedderFactory.d.ts +0 -4
- package/dist/src/services/EmbedderFactory.js +0 -5
- package/dist/src/services/IndexerService.d.ts +2 -10
- package/dist/src/services/IndexerService.js +29 -45
- package/dist/src/services/InfinityEmbedder.d.ts +0 -21
- package/dist/src/services/InfinityEmbedder.js +0 -15
- package/dist/src/services/LoggerService.d.ts +7 -5
- package/dist/src/services/LoggerService.js +37 -41
- package/dist/src/services/McpServer.d.ts +25 -10
- package/dist/src/services/McpServer.js +101 -64
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
- package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
- package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
- package/dist/src/services/NxMonorepoResolver.js +5 -56
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
- package/dist/src/services/OpenAICompatibleEmbedder.js +1 -14
- package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
- package/dist/src/services/ProjectDiscoveryService.js +0 -4
- package/dist/src/services/SearcherService.d.ts +127 -70
- package/dist/src/services/SearcherService.js +526 -305
- package/dist/src/services/VoyageEmbedder.d.ts +0 -6
- package/dist/src/services/VoyageEmbedder.js +0 -6
- package/dist/src/services/WebServerService.d.ts +1 -53
- package/dist/src/services/WebServerService.js +11 -57
- package/dist/src/services/http-retry.d.ts +0 -7
- package/dist/src/services/http-retry.js +1 -10
- package/dist/src/services/infinity-provisioning.d.ts +0 -74
- package/dist/src/services/infinity-provisioning.js +5 -88
- package/dist/src/services/legacy-ollama-index.d.ts +0 -16
- package/dist/src/services/legacy-ollama-index.js +0 -23
- package/dist/src/services/reranker.d.ts +0 -9
- package/dist/src/services/reranker.js +46 -12
- package/dist/src/setup.d.ts +11 -23
- package/dist/src/setup.js +98 -71
- package/dist/src/symbol-name.d.ts +0 -12
- package/dist/src/symbol-name.js +0 -13
- package/dist/src/symbol-rows.d.ts +2 -11
- package/dist/src/symbol-rows.js +5 -13
- package/dist/src/tool-call-log.d.ts +2 -0
- package/dist/src/tool-call-log.js +24 -0
- package/dist/src/tools/FindSymbolTool.js +14 -5
- package/dist/src/tools/GetCallersTool.d.ts +3 -1
- package/dist/src/tools/GetCallersTool.js +16 -6
- package/dist/src/tools/GetDependenciesTool.d.ts +3 -1
- package/dist/src/tools/GetDependenciesTool.js +15 -8
- package/dist/src/tools/GetDependentsTool.d.ts +3 -1
- package/dist/src/tools/GetDependentsTool.js +31 -16
- package/dist/src/tools/ListFilesTool.d.ts +3 -1
- package/dist/src/tools/ListFilesTool.js +14 -12
- package/dist/src/tools/ListProjectsTool.js +5 -1
- package/dist/src/tools/ReadFileRangeTool.d.ts +3 -1
- package/dist/src/tools/ReadFileRangeTool.js +12 -11
- package/dist/src/tools/ReadFileTool.d.ts +3 -1
- package/dist/src/tools/ReadFileTool.js +12 -10
- package/dist/src/tools/SearchCodebaseTool.js +7 -2
- package/dist/src/tools/format-helpers.d.ts +10 -1
- package/dist/src/tools/format-helpers.js +145 -8
- package/dist/src/web/dashboard.d.ts +0 -8
- package/dist/src/web/dashboard.js +7 -10
- package/package.json +10 -9
- package/dist/src/services/AstChunker.d.ts +0 -42
- package/dist/src/services/AstChunker.js +0 -158
- package/docs/web-dashboard.md +0 -206
|
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.SearcherService = void 0;
|
|
37
|
+
exports.rerankDocument = rerankDocument;
|
|
37
38
|
const fs = __importStar(require("fs"));
|
|
38
39
|
const path = __importStar(require("path"));
|
|
39
40
|
const readline = __importStar(require("readline"));
|
|
@@ -42,70 +43,327 @@ const file_utils_1 = require("../file-utils");
|
|
|
42
43
|
const MonorepoAwareDiscoveryService_1 = require("./MonorepoAwareDiscoveryService");
|
|
43
44
|
const legacy_ollama_index_1 = require("./legacy-ollama-index");
|
|
44
45
|
const symbol_name_1 = require("../symbol-name");
|
|
46
|
+
const dependency_traversal_1 = require("../dependency-traversal");
|
|
47
|
+
const result_shaping_1 = require("../search/result-shaping");
|
|
48
|
+
const hnsw_settings_1 = require("../hnsw-settings");
|
|
45
49
|
const SILENT_LOGGER = {
|
|
46
50
|
debug() { },
|
|
47
51
|
info() { },
|
|
48
52
|
warn() { },
|
|
49
53
|
error() { },
|
|
50
|
-
|
|
54
|
+
withContext(_, fn) { return fn(); },
|
|
51
55
|
};
|
|
56
|
+
const NODE_FS = {
|
|
57
|
+
realpath: p => fs.promises.realpath(p),
|
|
58
|
+
stat: p => fs.promises.stat(p),
|
|
59
|
+
readdir: p => fs.promises.readdir(p, { withFileTypes: true }),
|
|
60
|
+
createReadStream: p => fs.createReadStream(p, { encoding: 'utf-8' }),
|
|
61
|
+
};
|
|
62
|
+
const GRAPHQL_KINDS = new Set([
|
|
63
|
+
'graphql_query', 'graphql_mutation', 'graphql_subscription', 'graphql_field', 'graphql_type',
|
|
64
|
+
]);
|
|
52
65
|
const EMPTY_META = { description: null, stack: null, notes: null, relations: null };
|
|
53
66
|
const GREP_MAX_RESULTS = 200;
|
|
54
67
|
const GREP_RAW_LIMIT = GREP_MAX_RESULTS * 5;
|
|
55
68
|
const GREP_EXCLUDE_DIRS = [...file_utils_1.IGNORE_DIRS].map(d => `--exclude-dir=${d}`);
|
|
56
69
|
const FIND_SYMBOL_LIMIT = 50;
|
|
57
70
|
const FIND_SYMBOL_RAW_LIMIT = FIND_SYMBOL_LIMIT * 10;
|
|
71
|
+
const PATH_CANDIDATE_LIMIT = 3;
|
|
72
|
+
const INDEX_PATH_COLUMNS = [
|
|
73
|
+
{ table: 'code_chunk', column: '"filePath"', content: true },
|
|
74
|
+
{ table: 'code_symbol', column: '"filePath"', content: true },
|
|
75
|
+
{ table: 'code_dependency', column: '"fromFile"', content: true },
|
|
76
|
+
{ table: 'code_call', column: '"fromFile"', content: true },
|
|
77
|
+
{ table: 'code_dependency', column: '"toFile"', content: false },
|
|
78
|
+
{ table: 'code_call', column: '"calleeFile"', content: false },
|
|
79
|
+
];
|
|
80
|
+
const OUTSIDE_WORKSPACE_VALUE = (col) => `(${col} = '..' OR ${col} LIKE '../%')`;
|
|
81
|
+
function stripTrailingSlash(p) {
|
|
82
|
+
return p.length > 1 && p.endsWith('/') ? p.slice(0, -1) : p;
|
|
83
|
+
}
|
|
84
|
+
function isWithin(p, root) {
|
|
85
|
+
if (root === '.')
|
|
86
|
+
return true;
|
|
87
|
+
if (root === '/')
|
|
88
|
+
return p.startsWith('/');
|
|
89
|
+
return p === root || p.startsWith(root + '/');
|
|
90
|
+
}
|
|
91
|
+
function errorCode(err) {
|
|
92
|
+
return err?.code ?? 'EIO';
|
|
93
|
+
}
|
|
94
|
+
function rerankDocument(row) {
|
|
95
|
+
const symbol = row.ownerPath ? ` | Symbol: ${oneLine(row.ownerPath)}` : '';
|
|
96
|
+
return `File: ${oneLine(row.filePath)}${symbol}\n${row.content}`;
|
|
97
|
+
}
|
|
98
|
+
function oneLine(text) {
|
|
99
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
100
|
+
}
|
|
58
101
|
class SearcherService {
|
|
59
102
|
db;
|
|
60
103
|
embedder;
|
|
61
104
|
discovery;
|
|
62
105
|
config;
|
|
63
106
|
logger;
|
|
64
|
-
|
|
107
|
+
fs;
|
|
108
|
+
constructor(db, embedder, discovery, config, logger = SILENT_LOGGER, fsDeps = {}) {
|
|
65
109
|
this.db = db;
|
|
66
110
|
this.embedder = embedder;
|
|
67
111
|
this.discovery = discovery;
|
|
68
112
|
this.config = config;
|
|
69
113
|
this.logger = logger;
|
|
114
|
+
this.fs = { ...NODE_FS, ...fsDeps };
|
|
70
115
|
}
|
|
71
|
-
|
|
116
|
+
targetColumnPresent = false;
|
|
117
|
+
realRootCache;
|
|
72
118
|
rootMap;
|
|
73
|
-
|
|
119
|
+
pgvectorVersionRead;
|
|
120
|
+
degradedScanWarned = false;
|
|
121
|
+
async iterativeScanEnabled() {
|
|
122
|
+
const read = this.pgvectorVersionRead ??= this.db.pgvectorVersion().then(v => {
|
|
123
|
+
if (v === null)
|
|
124
|
+
throw new Error('vector extension not found in pg_extension');
|
|
125
|
+
return v;
|
|
126
|
+
});
|
|
127
|
+
let version;
|
|
128
|
+
try {
|
|
129
|
+
version = await read;
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
if (this.pgvectorVersionRead === read)
|
|
133
|
+
this.pgvectorVersionRead = undefined;
|
|
134
|
+
this.warnDegradedScan('unknown', err);
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
if ((0, hnsw_settings_1.supportsIterativeScan)(version))
|
|
138
|
+
return true;
|
|
139
|
+
this.warnDegradedScan(version);
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
warnDegradedScan(version, err) {
|
|
143
|
+
if (this.degradedScanWarned)
|
|
144
|
+
return;
|
|
145
|
+
this.degradedScanWarned = true;
|
|
146
|
+
this.logger.warn(`SearcherService: pgvector ${version} lacks hnsw.iterative_scan — filtered searches on large projects may return fewer candidates than requested; upgrade to pgvector ≥ 0.8`, err ? { err } : undefined);
|
|
147
|
+
}
|
|
74
148
|
projectRootMap() {
|
|
75
149
|
if (!this.rootMap) {
|
|
76
150
|
this.rootMap = new Map((0, MonorepoAwareDiscoveryService_1.toLeafProjects)(this.discovery.discover(this.config.workspaceRoot)).map(p => [p.name, p.path]));
|
|
77
151
|
}
|
|
78
152
|
return this.rootMap;
|
|
79
153
|
}
|
|
80
|
-
/**
|
|
81
|
-
* Resolve a project name to its real filesystem root. Uses discovery (leaf
|
|
82
|
-
* projects) so nested-monorepo names like `repo/app` map to their true path;
|
|
83
|
-
* falls back to `join(workspaceRoot, name)` when the name is unknown (e.g. a
|
|
84
|
-
* project since removed from disk, or a flat project not surfaced by discovery).
|
|
85
|
-
*/
|
|
86
154
|
resolveProjectRoot(name) {
|
|
87
155
|
let root = this.projectRootMap().get(name);
|
|
88
156
|
if (root === undefined) {
|
|
89
|
-
// Cache miss — the project may have been added/renamed on disk since the
|
|
90
|
-
// map was built (mcp/web servers are long-lived). Rebuild once and retry
|
|
91
|
-
// before the flat fallback, which is wrong for nested-monorepo names
|
|
92
|
-
// (`repo/app` lives at `repo/apps/app`, not `repo/app`).
|
|
93
157
|
this.rootMap = undefined;
|
|
94
158
|
root = this.projectRootMap().get(name);
|
|
95
159
|
}
|
|
96
160
|
return root ?? path.join(this.config.workspaceRoot, name);
|
|
97
161
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
162
|
+
toRel(absPath) {
|
|
163
|
+
return path.relative(this.config.workspaceRoot, absPath) || '.';
|
|
164
|
+
}
|
|
165
|
+
toAbs(relPath) {
|
|
166
|
+
return relPath === '.' ? this.workspaceRoot() : path.join(this.workspaceRoot(), relPath);
|
|
167
|
+
}
|
|
168
|
+
workspaceRoot() {
|
|
169
|
+
return stripTrailingSlash(path.posix.normalize(this.config.workspaceRoot));
|
|
170
|
+
}
|
|
171
|
+
async realWorkspaceRoot() {
|
|
172
|
+
if (this.realRootCache !== undefined)
|
|
173
|
+
return this.realRootCache;
|
|
174
|
+
try {
|
|
175
|
+
this.realRootCache = await this.fs.realpath(this.workspaceRoot());
|
|
176
|
+
return this.realRootCache;
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
this.logger.warn('SearcherService: workspaceRoot real path unavailable', { reason: errorCode(err) });
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async resolveToolPath(input) {
|
|
184
|
+
if (input == null || input === '')
|
|
185
|
+
return { ok: false, reason: 'invalid', missing: true };
|
|
186
|
+
if (input.includes('\\'))
|
|
187
|
+
return { ok: false, reason: 'invalid', missing: false };
|
|
188
|
+
if (path.posix.isAbsolute(input)) {
|
|
189
|
+
const abs = stripTrailingSlash(path.posix.normalize(input));
|
|
190
|
+
const realRoot = await this.realWorkspaceRoot();
|
|
191
|
+
for (const root of realRoot ? [this.workspaceRoot(), realRoot] : [this.workspaceRoot()]) {
|
|
192
|
+
if (!isWithin(abs, root))
|
|
193
|
+
continue;
|
|
194
|
+
const relative = abs === root ? '.' : abs.slice(root === '/' ? 1 : root.length + 1);
|
|
195
|
+
return { ok: false, reason: 'absolute', relative };
|
|
196
|
+
}
|
|
197
|
+
return { ok: false, reason: 'outside_workspace' };
|
|
198
|
+
}
|
|
199
|
+
const relPath = stripTrailingSlash(path.posix.normalize(input));
|
|
200
|
+
if (relPath === '..' || relPath.startsWith('../'))
|
|
201
|
+
return { ok: false, reason: 'outside_workspace' };
|
|
202
|
+
return { ok: true, relPath, absPath: this.toAbs(relPath) };
|
|
203
|
+
}
|
|
204
|
+
async realLocation(absPath) {
|
|
205
|
+
const root = (await this.realWorkspaceRoot()) ?? this.workspaceRoot();
|
|
206
|
+
let existing = absPath;
|
|
207
|
+
const rest = [];
|
|
208
|
+
for (;;) {
|
|
209
|
+
try {
|
|
210
|
+
const real = path.join(await this.fs.realpath(existing), ...rest);
|
|
211
|
+
return { ok: true, inside: isWithin(real, root) };
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
if (errorCode(err) !== 'ENOENT')
|
|
215
|
+
return { ok: false, code: errorCode(err) };
|
|
216
|
+
const parent = path.dirname(existing);
|
|
217
|
+
if (parent === existing)
|
|
218
|
+
return { ok: true, inside: false };
|
|
219
|
+
rest.unshift(path.basename(existing));
|
|
220
|
+
existing = parent;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
async checkRealLocation(relPath, absPath, kind) {
|
|
225
|
+
const loc = await this.realLocation(absPath);
|
|
226
|
+
if (!loc.ok)
|
|
227
|
+
return { ok: false, reason: 'unreadable', code: loc.code };
|
|
228
|
+
if (loc.inside)
|
|
229
|
+
return { ok: true };
|
|
230
|
+
const exempt = kind === 'file' ? await this.isContentPath(relPath) : await this.hasContentUnder(relPath);
|
|
231
|
+
return exempt ? { ok: true } : { ok: false, reason: 'outside_workspace' };
|
|
232
|
+
}
|
|
233
|
+
async pathExists(columns, predicate, param) {
|
|
234
|
+
const branches = columns.map(c => `SELECT 1 FROM ${c.table} WHERE ${predicate.replace(/\$col/g, c.column)}`);
|
|
235
|
+
const [{ exists }] = await this.db.query(`SELECT EXISTS (${branches.join(' UNION ALL ')}) AS exists`, [param]);
|
|
236
|
+
return exists;
|
|
237
|
+
}
|
|
238
|
+
isIndexedPath(relPath) {
|
|
239
|
+
return this.pathExists(INDEX_PATH_COLUMNS, '$col = $1', relPath);
|
|
240
|
+
}
|
|
241
|
+
isContentPath(relPath) {
|
|
242
|
+
return this.pathExists(INDEX_PATH_COLUMNS.filter(c => c.content), '$col = $1', relPath);
|
|
243
|
+
}
|
|
244
|
+
hasContentUnder(relPath) {
|
|
245
|
+
const prefix = relPath === '.' ? '' : (0, symbol_name_1.escapeLike)(relPath) + '/';
|
|
246
|
+
return this.pathExists(INDEX_PATH_COLUMNS.filter(c => c.content), `$col LIKE $1 ESCAPE '\\'`, prefix + '%');
|
|
247
|
+
}
|
|
248
|
+
async findPathCandidates(relPath, kind) {
|
|
249
|
+
const union = INDEX_PATH_COLUMNS.map(c => `SELECT ${c.column} AS p FROM ${c.table}`).join(' UNION ALL ');
|
|
250
|
+
const notOutside = `NOT ${OUTSIDE_WORKSPACE_VALUE('p')}`;
|
|
251
|
+
if (kind === 'file') {
|
|
252
|
+
const substituted = await this.substituteProjectName(relPath);
|
|
253
|
+
const rows = await this.db.query(`SELECT p FROM (${union}) s
|
|
254
|
+
WHERE p LIKE $1 ESCAPE '\\' AND ${notOutside}
|
|
255
|
+
GROUP BY p ORDER BY p COLLATE "C" LIMIT ${PATH_CANDIDATE_LIMIT + 1}`, ['%/' + (0, symbol_name_1.escapeLike)(relPath)]);
|
|
256
|
+
const first = substituted ? [substituted] : [];
|
|
257
|
+
return [...first, ...rows.map(r => r.p).filter(p => p !== substituted)].slice(0, PATH_CANDIDATE_LIMIT);
|
|
258
|
+
}
|
|
259
|
+
const projectRoot = this.findProjectRootByName(relPath);
|
|
260
|
+
const first = projectRoot && projectRoot !== relPath ? [projectRoot] : [];
|
|
261
|
+
const rows = await this.db.query(`SELECT d AS p FROM (
|
|
262
|
+
SELECT substring(p FROM 1 FOR position($2::text IN p) + length($3::text)) AS d
|
|
263
|
+
FROM (${union}) s
|
|
264
|
+
WHERE p LIKE $1 ESCAPE '\\' AND ${notOutside}
|
|
265
|
+
) dirs
|
|
266
|
+
GROUP BY d ORDER BY d COLLATE "C" LIMIT ${PATH_CANDIDATE_LIMIT + 1}`, ['%/' + (0, symbol_name_1.escapeLike)(relPath) + '/%', '/' + relPath + '/', relPath]);
|
|
267
|
+
const rest = rows.map(r => r.p).filter(p => !first.includes(p));
|
|
268
|
+
return [...first, ...rest].slice(0, PATH_CANDIDATE_LIMIT);
|
|
269
|
+
}
|
|
270
|
+
async candidatesOrNone(relPath, kind) {
|
|
271
|
+
try {
|
|
272
|
+
return await this.findPathCandidates(relPath, kind);
|
|
273
|
+
}
|
|
274
|
+
catch (err) {
|
|
275
|
+
this.logger.error('SearcherService: path candidate lookup failed', { relPath, err });
|
|
276
|
+
return [];
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
async substituteProjectName(relPath) {
|
|
280
|
+
let named = [];
|
|
281
|
+
for (let attempt = 0; attempt < 2 && named.length === 0; attempt++) {
|
|
282
|
+
if (attempt === 1)
|
|
283
|
+
this.rootMap = undefined;
|
|
284
|
+
named = [...this.projectRootMap().entries()].filter(([name]) => relPath.startsWith(name + '/'));
|
|
285
|
+
}
|
|
286
|
+
const longest = named.sort((a, b) => b[0].length - a[0].length)[0];
|
|
287
|
+
if (!longest)
|
|
288
|
+
return null;
|
|
289
|
+
const [name, absRoot] = longest;
|
|
290
|
+
const root = this.toRel(absRoot);
|
|
291
|
+
if (root === name)
|
|
292
|
+
return null;
|
|
293
|
+
const candidate = (root === '.' ? '' : root + '/') + relPath.slice(name.length + 1);
|
|
294
|
+
if (await this.isIndexedPath(candidate))
|
|
295
|
+
return candidate;
|
|
296
|
+
try {
|
|
297
|
+
return (await this.fs.stat(this.toAbs(candidate))).isFile() ? candidate : null;
|
|
298
|
+
}
|
|
299
|
+
catch {
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
findProjectRootByName(name) {
|
|
304
|
+
let root = this.projectRootMap().get(name);
|
|
305
|
+
if (root === undefined) {
|
|
306
|
+
this.rootMap = undefined;
|
|
307
|
+
root = this.projectRootMap().get(name);
|
|
308
|
+
}
|
|
309
|
+
return root === undefined ? null : this.toRel(root);
|
|
310
|
+
}
|
|
311
|
+
findProjectRoot(match) {
|
|
312
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
313
|
+
if (attempt === 1)
|
|
314
|
+
this.rootMap = undefined;
|
|
315
|
+
const best = [...this.projectRootMap().entries()]
|
|
316
|
+
.map(([name, abs]) => ({ name, rel: this.toRel(abs) }))
|
|
317
|
+
.filter(r => match(r.rel))
|
|
318
|
+
.sort((a, b) => b.rel.length - a.rel.length)[0];
|
|
319
|
+
if (best)
|
|
320
|
+
return best;
|
|
321
|
+
}
|
|
322
|
+
return null;
|
|
323
|
+
}
|
|
324
|
+
fallbackRoot(relPath) {
|
|
325
|
+
return this.findProjectRoot(r => isWithin(relPath, r))?.rel ?? null;
|
|
326
|
+
}
|
|
327
|
+
async checkGraphPath(relPath) {
|
|
328
|
+
if (await this.isIndexedPath(relPath))
|
|
329
|
+
return null;
|
|
330
|
+
const absPath = this.toAbs(relPath);
|
|
331
|
+
const loc = await this.realLocation(absPath);
|
|
332
|
+
if (!loc.ok)
|
|
333
|
+
return { ok: false, reason: 'unreadable', code: loc.code };
|
|
334
|
+
if (!loc.inside)
|
|
335
|
+
return { ok: false, reason: 'not_found', candidates: await this.candidatesOrNone(relPath, 'file') };
|
|
336
|
+
let stat;
|
|
337
|
+
try {
|
|
338
|
+
stat = await this.fs.stat(absPath);
|
|
339
|
+
}
|
|
340
|
+
catch (err) {
|
|
341
|
+
if (errorCode(err) !== 'ENOENT')
|
|
342
|
+
return { ok: false, reason: 'unreadable', code: errorCode(err) };
|
|
343
|
+
return { ok: false, reason: 'not_found', candidates: await this.candidatesOrNone(relPath, 'file') };
|
|
344
|
+
}
|
|
345
|
+
if (!stat.isFile())
|
|
346
|
+
return { ok: false, reason: 'not_a_file' };
|
|
347
|
+
return { ok: false, reason: 'not_indexed', explanation: await this.explainNotIndexed(relPath) };
|
|
348
|
+
}
|
|
349
|
+
async explainNotIndexed(relPath) {
|
|
350
|
+
const owner = this.findProjectRoot(r => r !== relPath && isWithin(relPath, r));
|
|
351
|
+
if (!owner)
|
|
352
|
+
return { kind: 'no_project' };
|
|
353
|
+
const inProject = owner.rel === '.' ? relPath : relPath.slice(owner.rel.length + 1);
|
|
354
|
+
const exclusion = (0, file_utils_1.indexingExclusion)(inProject, (0, file_utils_1.loadIgnore)(this.toAbs(owner.rel)));
|
|
355
|
+
if (exclusion)
|
|
356
|
+
return exclusion;
|
|
357
|
+
const [{ exists }] = await this.db.query(`SELECT EXISTS (
|
|
358
|
+
SELECT 1 FROM code_chunk WHERE "serviceName" = $1
|
|
359
|
+
UNION ALL SELECT 1 FROM code_symbol WHERE project = $1
|
|
360
|
+
UNION ALL SELECT 1 FROM code_dependency WHERE project = $1
|
|
361
|
+
UNION ALL SELECT 1 FROM code_call WHERE project = $1
|
|
362
|
+
) AS exists`, [owner.name]);
|
|
363
|
+
return exists ? { kind: 'unknown' } : { kind: 'project_not_indexed', project: owner.name };
|
|
364
|
+
}
|
|
105
365
|
async getProjectEdges() {
|
|
106
366
|
const rows = await this.db.query(`SELECT DISTINCT project, "toFile" FROM code_dependency`);
|
|
107
|
-
// Reverse map: workspace-relative project root → name, longest first so the
|
|
108
|
-
// most specific owner wins (e.g. repo/apps/account before repo).
|
|
109
367
|
const byRelPath = [...this.projectRootMap().entries()]
|
|
110
368
|
.map(([name, abs]) => ({ name, rel: path.relative(this.config.workspaceRoot, abs) }))
|
|
111
369
|
.sort((a, b) => b.rel.length - a.rel.length);
|
|
@@ -130,29 +388,15 @@ class SearcherService {
|
|
|
130
388
|
}
|
|
131
389
|
return edges;
|
|
132
390
|
}
|
|
133
|
-
// Query translation was removed (legacy). The query is embedded as-is; the
|
|
134
|
-
// method is kept so the tool layer needs no change and `translated` is always
|
|
135
|
-
// false.
|
|
136
391
|
async prepareQuery(raw) {
|
|
137
392
|
return { query: raw, translated: false };
|
|
138
393
|
}
|
|
139
|
-
/**
|
|
140
|
-
* Refuse when the configured embedding model disagrees with the one that built
|
|
141
|
-
* the index — embedding a query into a foreign vector space returns noise. A
|
|
142
|
-
* missing `index_meta` row with an empty (or altogether absent) `code_chunk`
|
|
143
|
-
* does not block — but a *populated* `code_chunk` with no `index_meta` row is a
|
|
144
|
-
* pre-`index_meta` install, treated as the legacy `ollama`/`bge-m3` index (same
|
|
145
|
-
* interpretation as the `rag-index` legacy-Ollama guard), read-only — this check
|
|
146
|
-
* never writes `index_meta`.
|
|
147
|
-
*/
|
|
148
394
|
async assertIndexMatches() {
|
|
149
395
|
const meta = await this.db.readIndexMeta();
|
|
150
396
|
if (meta) {
|
|
151
397
|
this.compareIndexed(meta.provider, meta.model, meta.dimensions);
|
|
152
398
|
return;
|
|
153
399
|
}
|
|
154
|
-
// No row — a missing code_chunk table counts as empty; any other DB error
|
|
155
|
-
// while counting propagates rather than being read as "not indexed".
|
|
156
400
|
if (await (0, legacy_ollama_index_1.hasIndexedChunks)(this.db)) {
|
|
157
401
|
this.compareIndexed(legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.provider, legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.model, null);
|
|
158
402
|
}
|
|
@@ -163,14 +407,6 @@ class SearcherService {
|
|
|
163
407
|
throw new Error(`Index was built with ${indexedProvider}/${indexedModel} but the config selects ${provider}/${model}. ` +
|
|
164
408
|
`Run \`rag-index --reindex\` to rebuild the index for the new model.`);
|
|
165
409
|
}
|
|
166
|
-
// Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
|
|
167
|
-
// override) would embed the query at a width the column can't hold — pgvector
|
|
168
|
-
// would throw an opaque "different vector dimensions" error mid-query. Catch
|
|
169
|
-
// it here with the friendly hint. `embedder.dimensions` is 0 when the width is
|
|
170
|
-
// only knowable by probing, in which case we skip (can't compare cheaply).
|
|
171
|
-
// `indexedDimensions` is null for the legacy-inferred identity, whose provider
|
|
172
|
-
// can never match a configured provider (ollama is no longer selectable), so
|
|
173
|
-
// this branch is unreached in that case.
|
|
174
410
|
const dim = this.embedder.dimensions;
|
|
175
411
|
if (indexedDimensions != null && dim > 0 && dim !== indexedDimensions) {
|
|
176
412
|
throw new Error(`Index was built at ${indexedDimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
|
|
@@ -182,28 +418,36 @@ class SearcherService {
|
|
|
182
418
|
FROM code_chunk
|
|
183
419
|
GROUP BY "serviceName"
|
|
184
420
|
ORDER BY "serviceName"`);
|
|
185
|
-
return rows.map(r => {
|
|
421
|
+
return Promise.all(rows.map(async (r) => {
|
|
186
422
|
const projectRoot = this.resolveProjectRoot(r.serviceName);
|
|
187
423
|
const meta = (0, file_utils_1.readRagMeta)(projectRoot);
|
|
188
|
-
|
|
189
|
-
|
|
424
|
+
let rootError = null;
|
|
425
|
+
try {
|
|
426
|
+
await this.fs.stat(projectRoot);
|
|
427
|
+
}
|
|
428
|
+
catch (err) {
|
|
429
|
+
rootError = errorCode(err);
|
|
430
|
+
}
|
|
431
|
+
return {
|
|
432
|
+
...r, path: projectRoot, relPath: this.toRel(projectRoot), rootError,
|
|
433
|
+
...(typeof meta === 'object' ? meta : EMPTY_META),
|
|
434
|
+
};
|
|
435
|
+
}));
|
|
190
436
|
}
|
|
191
|
-
async searchChunks(query, limit, project, threshold) {
|
|
437
|
+
async searchChunks(query, limit, project, threshold, rerankOverride, perFile = result_shaping_1.DEFAULT_PER_FILE) {
|
|
438
|
+
const { stages } = await this.searchChunksWithStages(query, limit, project, threshold, rerankOverride, perFile);
|
|
439
|
+
return stages.reranked ?? stages.vector;
|
|
440
|
+
}
|
|
441
|
+
async searchChunksWithStages(query, limit, project, threshold, rerankOverride, perFile = result_shaping_1.DEFAULT_PER_FILE) {
|
|
192
442
|
const t0 = Date.now();
|
|
193
443
|
await this.assertIndexMatches();
|
|
194
444
|
const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
|
|
195
|
-
const reranking = !!this.embedder.rerank;
|
|
196
|
-
|
|
197
|
-
// (e.g. the default 5) still gives the reranker a large-enough menu. Without
|
|
198
|
-
// a reranker the extra candidates are never reordered, so keep `limit`.
|
|
445
|
+
const reranking = rerankOverride === false ? false : !!this.embedder.rerank;
|
|
446
|
+
const perFileActive = perFile < limit;
|
|
199
447
|
const fetchLimit = reranking
|
|
200
448
|
? Math.min(Math.max(limit * this.config.reranker.overfetch, this.config.reranker.candidateFloor), 200)
|
|
201
|
-
: limit;
|
|
202
|
-
|
|
203
|
-
// under-threshold candidates existed than the window took). The extra row is
|
|
204
|
-
// dropped before reranking so it never affects results.
|
|
205
|
-
const metricsOn = this.config.logging.searchMetrics;
|
|
206
|
-
const dbLimit = metricsOn ? fetchLimit + 1 : fetchLimit;
|
|
449
|
+
: perFileActive ? Math.min(limit * 3, 200) : limit;
|
|
450
|
+
const dbLimit = fetchLimit + 1;
|
|
207
451
|
this.logger.debug('SearcherService: searchChunks', {
|
|
208
452
|
query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
|
|
209
453
|
});
|
|
@@ -216,8 +460,14 @@ class SearcherService {
|
|
|
216
460
|
params.push(project);
|
|
217
461
|
projectFilter = `AND "serviceName" = $${params.length}`;
|
|
218
462
|
}
|
|
463
|
+
const iterative = await this.iterativeScanEnabled();
|
|
464
|
+
const settings = (0, hnsw_settings_1.hnswSettings)(dbLimit, iterative);
|
|
465
|
+
const indexParams = {
|
|
466
|
+
efSearch: settings['hnsw.ef_search'],
|
|
467
|
+
iterativeScan: iterative ? hnsw_settings_1.ITERATIVE_SCAN_MODE : 'off',
|
|
468
|
+
};
|
|
219
469
|
const tDb0 = Date.now();
|
|
220
|
-
const fetched = await this.db.
|
|
470
|
+
const fetched = await this.db.queryWithLocalSettings(settings, `SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
|
|
221
471
|
ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
|
|
222
472
|
FROM code_chunk
|
|
223
473
|
WHERE embedding <=> $1 < $2
|
|
@@ -225,9 +475,8 @@ class SearcherService {
|
|
|
225
475
|
ORDER BY embedding <=> $1
|
|
226
476
|
LIMIT $3`, params);
|
|
227
477
|
const dbMs = Date.now() - tDb0;
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
const poolSaturated = metricsOn && fetched.length > fetchLimit;
|
|
478
|
+
fetched.sort((a, b) => a.score - b.score);
|
|
479
|
+
const poolSaturated = fetched.length > fetchLimit;
|
|
231
480
|
const candidates = fetched.slice(0, fetchLimit);
|
|
232
481
|
this.logger.debug('SearcherService: searchChunks candidates', {
|
|
233
482
|
count: candidates.length,
|
|
@@ -235,45 +484,45 @@ class SearcherService {
|
|
|
235
484
|
files: candidates.map(r => r.filePath),
|
|
236
485
|
ms: Date.now() - t0,
|
|
237
486
|
});
|
|
238
|
-
|
|
487
|
+
const vector = (0, result_shaping_1.limitPerFile)(candidates, limit, perFile);
|
|
488
|
+
let reranked;
|
|
489
|
+
let rerankFailed = false;
|
|
239
490
|
let rerankMs = 0;
|
|
240
|
-
if (
|
|
241
|
-
|
|
491
|
+
if (reranking && candidates.length === 0) {
|
|
492
|
+
reranked = [];
|
|
242
493
|
}
|
|
243
|
-
else {
|
|
494
|
+
else if (reranking) {
|
|
244
495
|
const tRerank0 = Date.now();
|
|
245
496
|
try {
|
|
246
|
-
const
|
|
497
|
+
const topN = perFileActive ? candidates.length : limit;
|
|
498
|
+
const indices = await this.embedder.rerank(query, candidates.map(rerankDocument), topN);
|
|
247
499
|
rerankMs = Date.now() - tRerank0;
|
|
248
500
|
this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
results = indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
|
|
501
|
+
const order = indices.filter(i => i >= 0 && i < candidates.length).map(i => candidates[i]);
|
|
502
|
+
reranked = (0, result_shaping_1.limitPerFile)(order, limit, perFile);
|
|
252
503
|
}
|
|
253
504
|
catch (err) {
|
|
254
505
|
rerankMs = Date.now() - tRerank0;
|
|
255
506
|
this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
|
|
256
|
-
|
|
507
|
+
err,
|
|
257
508
|
ms: Date.now() - t0,
|
|
258
509
|
});
|
|
259
|
-
|
|
510
|
+
reranked = vector;
|
|
511
|
+
rerankFailed = true;
|
|
260
512
|
}
|
|
261
513
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
514
|
+
this.emitSearchMetrics(query, {
|
|
515
|
+
fetchLimit, candidates, results: reranked ?? vector, poolSaturated, indexParams,
|
|
516
|
+
embedMs, dbMs, rerankMs, totalMs: Date.now() - t0,
|
|
517
|
+
});
|
|
518
|
+
return {
|
|
519
|
+
pool: candidates,
|
|
520
|
+
stages: reranked ? { vector, reranked } : { vector },
|
|
521
|
+
rerankFailed,
|
|
522
|
+
fetchLimit,
|
|
523
|
+
indexParams,
|
|
524
|
+
};
|
|
268
525
|
}
|
|
269
|
-
/**
|
|
270
|
-
* Emit one structured search-tuning line (via `logger.metric`, at `info`
|
|
271
|
-
* regardless of `logging.level`). `candidates` are in cosine order, so a
|
|
272
|
-
* candidate's array index is its cosine rank — `maxCosineRankSurvived` is the
|
|
273
|
-
* deepest cosine position that survived reranking into the returned set. Read
|
|
274
|
-
* it alongside `poolSaturated` to tune `candidateFloor`/`overfetch`/threshold
|
|
275
|
-
* per hardware. Never logs chunk content, embeddings, or credentials.
|
|
276
|
-
*/
|
|
277
526
|
emitSearchMetrics(query, m) {
|
|
278
527
|
const { candidates, results } = m;
|
|
279
528
|
const contentIndex = new Map(candidates.map((c, i) => [c, i]));
|
|
@@ -282,7 +531,7 @@ class SearcherService {
|
|
|
282
531
|
for (const r of results)
|
|
283
532
|
fileCounts.set(r.filePath, (fileCounts.get(r.filePath) ?? 0) + 1);
|
|
284
533
|
const topFileCount = results.length ? Math.max(...fileCounts.values()) : 0;
|
|
285
|
-
this.logger.
|
|
534
|
+
this.logger.info('search metrics', {
|
|
286
535
|
query: query.slice(0, 120),
|
|
287
536
|
fetchLimit: m.fetchLimit,
|
|
288
537
|
candidatesReturned: candidates.length,
|
|
@@ -293,6 +542,8 @@ class SearcherService {
|
|
|
293
542
|
topFileShare: results.length ? +(topFileCount / results.length).toFixed(3) : 0,
|
|
294
543
|
bestResultScore: results.length ? Math.min(...results.map(r => r.score)) : null,
|
|
295
544
|
worstResultScore: results.length ? Math.max(...results.map(r => r.score)) : null,
|
|
545
|
+
efSearch: m.indexParams.efSearch,
|
|
546
|
+
iterativeScan: m.indexParams.iterativeScan,
|
|
296
547
|
embedMs: m.embedMs,
|
|
297
548
|
rerankMs: m.rerankMs,
|
|
298
549
|
dbMs: m.dbMs,
|
|
@@ -335,38 +586,27 @@ class SearcherService {
|
|
|
335
586
|
}
|
|
336
587
|
async fetchGraphContext(results) {
|
|
337
588
|
const t0 = Date.now();
|
|
338
|
-
const
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
if (!seen.has(key))
|
|
342
|
-
seen.set(key, { filePath: r.filePath, project: r.serviceName });
|
|
343
|
-
}
|
|
344
|
-
const pairs = [...seen.values()];
|
|
345
|
-
this.logger.debug('SearcherService: fetchGraphContext', { files: pairs.length });
|
|
346
|
-
if (pairs.length === 0)
|
|
589
|
+
const files = [...new Set(results.map(r => r.filePath))];
|
|
590
|
+
this.logger.debug('SearcherService: fetchGraphContext', { files: files.length });
|
|
591
|
+
if (files.length === 0)
|
|
347
592
|
return new Map();
|
|
348
|
-
const
|
|
349
|
-
|
|
350
|
-
const proportionalLimit = Math.max(2000, n * 100);
|
|
351
|
-
const firstHalf = pairs.map((_, i) => `($${i * 2 + 1}::text, $${i * 2 + 2}::text)`).join(', ');
|
|
352
|
-
const secondHalf = pairs.map((_, i) => `($${n * 2 + i * 2 + 1}::text, $${n * 2 + i * 2 + 2}::text)`).join(', ');
|
|
353
|
-
const params = [
|
|
354
|
-
...pairs.flatMap(p => [p.filePath, p.project]),
|
|
355
|
-
...pairs.flatMap(p => [p.filePath, p.project]),
|
|
356
|
-
];
|
|
593
|
+
const proportionalLimit = Math.max(2000, files.length * 100);
|
|
594
|
+
const values = files.map((_, i) => `($${i + 1}::text)`).join(', ');
|
|
357
595
|
const rows = await this.db.query(`SELECT r.fp AS "filePath", 'import' AS direction, d."toFile" AS related
|
|
358
|
-
FROM (VALUES ${
|
|
359
|
-
JOIN code_dependency d ON d."fromFile" = r.fp
|
|
596
|
+
FROM (VALUES ${values}) AS r(fp)
|
|
597
|
+
JOIN code_dependency d ON d."fromFile" = r.fp
|
|
598
|
+
WHERE NOT ${OUTSIDE_WORKSPACE_VALUE('d."toFile"')}
|
|
360
599
|
|
|
361
600
|
UNION ALL
|
|
362
601
|
|
|
363
602
|
SELECT r.fp AS "filePath", 'usedby' AS direction, d."fromFile" AS related
|
|
364
|
-
FROM (VALUES ${
|
|
365
|
-
JOIN code_dependency d ON d."toFile" = r.fp
|
|
603
|
+
FROM (VALUES ${values}) AS r(fp)
|
|
604
|
+
JOIN code_dependency d ON d."toFile" = r.fp
|
|
605
|
+
WHERE NOT ${OUTSIDE_WORKSPACE_VALUE('d."fromFile"')}
|
|
366
606
|
|
|
367
|
-
LIMIT ${proportionalLimit}`,
|
|
607
|
+
LIMIT ${proportionalLimit}`, files);
|
|
368
608
|
const ctx = new Map();
|
|
369
|
-
for (const
|
|
609
|
+
for (const filePath of files)
|
|
370
610
|
ctx.set(filePath, { imports: new Set(), usedBy: new Set() });
|
|
371
611
|
for (const row of rows) {
|
|
372
612
|
const g = ctx.get(row.filePath);
|
|
@@ -375,7 +615,7 @@ class SearcherService {
|
|
|
375
615
|
else
|
|
376
616
|
g.usedBy.add(row.related);
|
|
377
617
|
}
|
|
378
|
-
this.logger.debug('SearcherService: fetchGraphContext result', { files:
|
|
618
|
+
this.logger.debug('SearcherService: fetchGraphContext result', { files: files.length, rows: rows.length, ms: Date.now() - t0 });
|
|
379
619
|
const result = new Map();
|
|
380
620
|
for (const [fp, g] of ctx)
|
|
381
621
|
result.set(fp, {
|
|
@@ -384,89 +624,65 @@ class SearcherService {
|
|
|
384
624
|
});
|
|
385
625
|
return result;
|
|
386
626
|
}
|
|
387
|
-
async getDependencies(filePath,
|
|
627
|
+
async getDependencies(filePath, symbol = null, transitive = false) {
|
|
388
628
|
const t0 = Date.now();
|
|
389
|
-
this.logger.debug('SearcherService: getDependencies', { filePath,
|
|
390
|
-
const params = [filePath
|
|
629
|
+
this.logger.debug('SearcherService: getDependencies', { filePath, symbol, transitive });
|
|
630
|
+
const params = [filePath];
|
|
391
631
|
let symbolFilter = '';
|
|
392
632
|
if (symbol) {
|
|
393
633
|
params.push(symbol);
|
|
394
634
|
symbolFilter = `AND "toSymbol" = $${params.length}`;
|
|
395
635
|
}
|
|
396
|
-
const direct = await this.db.query(`SELECT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
|
|
636
|
+
const direct = await this.db.query(`SELECT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType", min("id"::text) AS "id"
|
|
397
637
|
FROM code_dependency
|
|
398
|
-
WHERE "fromFile" = $1 AND
|
|
638
|
+
WHERE "fromFile" = $1 AND NOT ${OUTSIDE_WORKSPACE_VALUE('"toFile"')}
|
|
399
639
|
${symbolFilter}
|
|
400
|
-
|
|
640
|
+
GROUP BY "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
|
|
641
|
+
ORDER BY "toFile", min("id"::text)`, params);
|
|
401
642
|
if (!transitive) {
|
|
402
643
|
this.logger.debug('SearcherService: getDependencies result', { direct: direct.length, ms: Date.now() - t0 });
|
|
403
644
|
return direct;
|
|
404
645
|
}
|
|
405
|
-
const
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
JOIN deps ON deps."toFile" = d."fromFile"
|
|
415
|
-
WHERE d.project = $2
|
|
416
|
-
AND d."toFile" <> $1
|
|
417
|
-
AND deps.depth < 20
|
|
418
|
-
)
|
|
419
|
-
SELECT "toFile", MIN(depth) AS depth,
|
|
420
|
-
(array_agg(via ORDER BY depth))[1] AS via,
|
|
421
|
-
NULL::text AS "fromFile", NULL::text AS "fromSymbol",
|
|
422
|
-
NULL::text AS "toSymbol", NULL::text AS "depType"
|
|
423
|
-
FROM deps
|
|
424
|
-
WHERE depth > 1
|
|
425
|
-
GROUP BY "toFile"
|
|
426
|
-
ORDER BY MIN(depth), "toFile"`, [filePath, project]);
|
|
646
|
+
const neighbours = async (frontier) => {
|
|
647
|
+
const rows = await this.db.query(`SELECT DISTINCT "fromFile", "toFile" FROM code_dependency
|
|
648
|
+
WHERE "fromFile" = ANY($1::text[]) AND NOT ${OUTSIDE_WORKSPACE_VALUE('"toFile"')}`, [frontier]);
|
|
649
|
+
return rows.map(r => ({ from: r.fromFile, to: r.toFile }));
|
|
650
|
+
};
|
|
651
|
+
const indirect = (await (0, dependency_traversal_1.traverse)(filePath, neighbours)).map(r => ({
|
|
652
|
+
toFile: r.file, depth: r.depth, via: r.via,
|
|
653
|
+
fromFile: null, fromSymbol: null, toSymbol: null, depType: null,
|
|
654
|
+
}));
|
|
427
655
|
this.logger.debug('SearcherService: getDependencies result', { direct: direct.length, indirect: indirect.length, ms: Date.now() - t0 });
|
|
428
656
|
return [...direct, ...indirect];
|
|
429
657
|
}
|
|
430
|
-
async getDependents(filePath,
|
|
658
|
+
async getDependents(filePath, symbol = null, transitive = false) {
|
|
431
659
|
const t0 = Date.now();
|
|
432
|
-
this.logger.debug('SearcherService: getDependents', { filePath,
|
|
433
|
-
const params = [filePath
|
|
660
|
+
this.logger.debug('SearcherService: getDependents', { filePath, symbol, transitive });
|
|
661
|
+
const params = [filePath];
|
|
434
662
|
let symbolFilter = '';
|
|
435
663
|
if (symbol) {
|
|
436
664
|
params.push(symbol);
|
|
437
665
|
symbolFilter = `AND "toSymbol" = $${params.length}`;
|
|
438
666
|
}
|
|
439
|
-
const direct = await this.db.query(`SELECT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
|
|
667
|
+
const direct = await this.db.query(`SELECT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType", min("id"::text) AS "id"
|
|
440
668
|
FROM code_dependency
|
|
441
|
-
WHERE "toFile" = $1 AND
|
|
669
|
+
WHERE "toFile" = $1 AND NOT ${OUTSIDE_WORKSPACE_VALUE('"fromFile"')}
|
|
442
670
|
${symbolFilter}
|
|
443
|
-
|
|
671
|
+
GROUP BY "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
|
|
672
|
+
ORDER BY "fromFile", min("id"::text)`, params);
|
|
444
673
|
if (!transitive) {
|
|
445
674
|
this.logger.debug('SearcherService: getDependents result', { direct: direct.length, ms: Date.now() - t0 });
|
|
446
675
|
return direct;
|
|
447
676
|
}
|
|
448
|
-
const
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
JOIN rdeps ON rdeps."fromFile" = d."toFile"
|
|
458
|
-
WHERE d.project = $2
|
|
459
|
-
AND d."fromFile" <> $1
|
|
460
|
-
AND rdeps.depth < 20
|
|
461
|
-
)
|
|
462
|
-
SELECT "fromFile", MIN(depth) AS depth,
|
|
463
|
-
(array_agg(via ORDER BY depth))[1] AS via,
|
|
464
|
-
NULL::text AS "toFile", NULL::text AS "fromSymbol",
|
|
465
|
-
NULL::text AS "toSymbol", NULL::text AS "depType"
|
|
466
|
-
FROM rdeps
|
|
467
|
-
WHERE depth > 1
|
|
468
|
-
GROUP BY "fromFile"
|
|
469
|
-
ORDER BY MIN(depth), "fromFile"`, [filePath, project]);
|
|
677
|
+
const neighbours = async (frontier) => {
|
|
678
|
+
const rows = await this.db.query(`SELECT DISTINCT "fromFile", "toFile" FROM code_dependency
|
|
679
|
+
WHERE "toFile" = ANY($1::text[]) AND NOT ${OUTSIDE_WORKSPACE_VALUE('"fromFile"')}`, [frontier]);
|
|
680
|
+
return rows.map(r => ({ from: r.toFile, to: r.fromFile }));
|
|
681
|
+
};
|
|
682
|
+
const indirect = (await (0, dependency_traversal_1.traverse)(filePath, neighbours)).map(r => ({
|
|
683
|
+
fromFile: r.file, depth: r.depth, via: r.via,
|
|
684
|
+
toFile: null, fromSymbol: null, toSymbol: null, depType: null,
|
|
685
|
+
}));
|
|
470
686
|
this.logger.debug('SearcherService: getDependents result', { direct: direct.length, indirect: indirect.length, ms: Date.now() - t0 });
|
|
471
687
|
return [...direct, ...indirect];
|
|
472
688
|
}
|
|
@@ -503,42 +719,21 @@ class SearcherService {
|
|
|
503
719
|
this.logger.debug('SearcherService: getCallers result', { count: rows.length, ms: Date.now() - t0 });
|
|
504
720
|
return rows;
|
|
505
721
|
}
|
|
506
|
-
|
|
507
|
-
* One-line warning when any project in `scope` (the given `project`, or
|
|
508
|
-
* every project with `code_chunk`/`code_symbol` rows) is outdated — shown
|
|
509
|
-
* first in `find_symbol`/`get_callers` output. Any failure to determine the
|
|
510
|
-
* scope or read markers only drops the notice (warn logged); the lookup
|
|
511
|
-
* itself is unaffected.
|
|
512
|
-
*/
|
|
513
|
-
async outdatedNotice(project) {
|
|
514
|
-
try {
|
|
515
|
-
const outdated = await this.db.listOutdatedProjects(project ? [project] : null);
|
|
516
|
-
if (outdated.length === 0)
|
|
517
|
-
return null;
|
|
518
|
-
return `⚠ Indexed before member indexing: ${[...outdated].sort().join(', ')} — run rag-index --reindex; methods and qualified names may be missing.`;
|
|
519
|
-
}
|
|
520
|
-
catch (err) {
|
|
521
|
-
this.logger.warn('SearcherService: failed to determine outdated-index scope', { reason: err.message });
|
|
522
|
-
return null;
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
async grepSymbol(symbol, project, excludeFile) {
|
|
722
|
+
async grepSymbol(symbol, rootDir, excludeFile) {
|
|
526
723
|
const t0 = Date.now();
|
|
527
|
-
this.logger.debug('SearcherService: grepSymbol', { symbol,
|
|
528
|
-
const projectDir = this.resolveProjectRoot(project);
|
|
724
|
+
this.logger.debug('SearcherService: grepSymbol', { symbol, rootDir, excludeFile });
|
|
529
725
|
const result = await this.spawnGrep([
|
|
530
726
|
'-rn', '-F', '-w',
|
|
531
727
|
'-m', String(GREP_RAW_LIMIT),
|
|
532
728
|
'--include=*.ts', '--include=*.tsx', '--include=*.js', '--include=*.jsx',
|
|
533
729
|
...GREP_EXCLUDE_DIRS,
|
|
534
730
|
symbol,
|
|
535
|
-
|
|
731
|
+
this.toAbs(rootDir),
|
|
536
732
|
]);
|
|
537
|
-
if (!result.ok
|
|
538
|
-
return {
|
|
539
|
-
}
|
|
733
|
+
if (!result.ok && result.output === null)
|
|
734
|
+
return { ok: false };
|
|
540
735
|
const excludeAbsPath = path.join(this.config.workspaceRoot, excludeFile);
|
|
541
|
-
const all = result.output.split('\n').flatMap(line => {
|
|
736
|
+
const all = (result.output ?? '').split('\n').flatMap(line => {
|
|
542
737
|
const match = line.match(/^(.+?):(\d+):/);
|
|
543
738
|
if (!match)
|
|
544
739
|
return [];
|
|
@@ -548,22 +743,34 @@ class SearcherService {
|
|
|
548
743
|
return [{ filePath: path.relative(this.config.workspaceRoot, absFile), line: parseInt(lineNum, 10) }];
|
|
549
744
|
});
|
|
550
745
|
const truncated = all.length > GREP_MAX_RESULTS;
|
|
551
|
-
this.logger.debug('SearcherService: grepSymbol result', { matches: Math.min(all.length, GREP_MAX_RESULTS), truncated, ms: Date.now() - t0 });
|
|
552
|
-
return { matches: all.slice(0, GREP_MAX_RESULTS), truncated };
|
|
746
|
+
this.logger.debug('SearcherService: grepSymbol result', { matches: Math.min(all.length, GREP_MAX_RESULTS), truncated, partial: !result.ok, ms: Date.now() - t0 });
|
|
747
|
+
return { ok: true, matches: all.slice(0, GREP_MAX_RESULTS), truncated, partial: !result.ok };
|
|
553
748
|
}
|
|
554
|
-
// Sync wrapper: validates name (throws synchronously), then returns Promise.
|
|
555
|
-
// Every kind, including 'method', is looked up in the index first.
|
|
556
749
|
findSymbol(name, project, kind) {
|
|
557
750
|
const parsed = (0, symbol_name_1.parseFindSymbolName)(name);
|
|
558
751
|
if (!parsed)
|
|
559
752
|
throw new Error('Invalid symbol name');
|
|
560
753
|
return this.findSymbolFromIndex(parsed, project, kind);
|
|
561
754
|
}
|
|
755
|
+
async hasTargetColumn() {
|
|
756
|
+
if (this.targetColumnPresent)
|
|
757
|
+
return true;
|
|
758
|
+
const rows = await this.db.query(`SELECT EXISTS (
|
|
759
|
+
SELECT 1 FROM information_schema.columns
|
|
760
|
+
WHERE table_schema = 'public' AND table_name = 'code_symbol' AND column_name = 'target'
|
|
761
|
+
) AS present`);
|
|
762
|
+
const present = !!rows[0]?.present;
|
|
763
|
+
if (present)
|
|
764
|
+
this.targetColumnPresent = true;
|
|
765
|
+
return present;
|
|
766
|
+
}
|
|
562
767
|
async findSymbolFromIndex(parsed, project, kind) {
|
|
563
768
|
const t0 = Date.now();
|
|
564
769
|
this.logger.debug('SearcherService: findSymbol', { parsed, project, kind, source: 'index' });
|
|
565
770
|
const container = 'container' in parsed ? parsed.container : null;
|
|
771
|
+
const isGraphqlKind = GRAPHQL_KINDS.has(kind);
|
|
566
772
|
try {
|
|
773
|
+
const hasTarget = await this.hasTargetColumn();
|
|
567
774
|
const params = [parsed.name];
|
|
568
775
|
let nameFilter = 'name = $1';
|
|
569
776
|
let orderPrefix = '';
|
|
@@ -586,10 +793,11 @@ class SearcherService {
|
|
|
586
793
|
kindFilter = `AND kind = $${params.length}`;
|
|
587
794
|
}
|
|
588
795
|
params.push(FIND_SYMBOL_LIMIT + 1);
|
|
589
|
-
const
|
|
796
|
+
const targetSelect = hasTarget ? '"target"' : 'NULL AS target';
|
|
797
|
+
const rows = await this.db.query(`SELECT "filePath", name, kind, signature, line, col, container, "isStatic", ${targetSelect}
|
|
590
798
|
FROM code_symbol
|
|
591
799
|
WHERE ${nameFilter} ${projectFilter} ${kindFilter}
|
|
592
|
-
ORDER BY ${orderPrefix}project, "filePath", line
|
|
800
|
+
ORDER BY ${orderPrefix}project, "filePath", line, col, kind COLLATE "C", container COLLATE "C" NULLS FIRST
|
|
593
801
|
LIMIT $${params.length}`, params);
|
|
594
802
|
const truncated = rows.length > FIND_SYMBOL_LIMIT;
|
|
595
803
|
const hits = rows.slice(0, FIND_SYMBOL_LIMIT).map(row => ({
|
|
@@ -601,16 +809,21 @@ class SearcherService {
|
|
|
601
809
|
qualified: row.container ? `${row.container}.${row.name}` : row.name,
|
|
602
810
|
isStatic: !!row.isStatic,
|
|
603
811
|
source: 'index',
|
|
812
|
+
target: row.target ?? null,
|
|
604
813
|
}));
|
|
605
814
|
this.logger.debug('SearcherService: findSymbol result', { source: 'index', hits: hits.length, truncated, ms: Date.now() - t0 });
|
|
606
815
|
if (hits.length > 0)
|
|
607
816
|
return { hits, truncated, grepFailed: false };
|
|
817
|
+
if (isGraphqlKind)
|
|
818
|
+
return { hits: [], truncated: false, grepFailed: false };
|
|
608
819
|
if (parsed.form === 'quoted' || parsed.form === 'computed')
|
|
609
820
|
return { hits: [], truncated: false, grepFailed: false };
|
|
610
821
|
return this.grepFindSymbol(parsed, project, kind);
|
|
611
822
|
}
|
|
612
823
|
catch (err) {
|
|
613
|
-
this.logger.warn('SearcherService: symbol index unavailable
|
|
824
|
+
this.logger.warn('SearcherService: symbol index unavailable', { err, ms: Date.now() - t0 });
|
|
825
|
+
if (isGraphqlKind)
|
|
826
|
+
throw err;
|
|
614
827
|
if (parsed.form === 'quoted' || parsed.form === 'computed')
|
|
615
828
|
return { hits: [], truncated: false, grepFailed: false };
|
|
616
829
|
return this.grepFindSymbol(parsed, project, kind);
|
|
@@ -618,7 +831,7 @@ class SearcherService {
|
|
|
618
831
|
}
|
|
619
832
|
async grepFindSymbol(parsed, project, kind) {
|
|
620
833
|
const t0 = Date.now();
|
|
621
|
-
const grepName = parsed.name;
|
|
834
|
+
const grepName = parsed.name;
|
|
622
835
|
this.logger.debug('SearcherService: findSymbol', { grepName, project, kind, source: 'grep' });
|
|
623
836
|
const pattern = this.buildFindSymbolPattern(grepName, kind);
|
|
624
837
|
if (pattern === null)
|
|
@@ -661,124 +874,136 @@ class SearcherService {
|
|
|
661
874
|
qualified: grepName,
|
|
662
875
|
isStatic: false,
|
|
663
876
|
source: 'grep',
|
|
877
|
+
target: null,
|
|
664
878
|
});
|
|
665
879
|
}
|
|
666
880
|
const truncated = parsedHits.length > FIND_SYMBOL_LIMIT;
|
|
667
881
|
this.logger.debug('SearcherService: findSymbol result', { source: 'grep', hits: Math.min(parsedHits.length, FIND_SYMBOL_LIMIT), truncated, ms: Date.now() - t0 });
|
|
668
882
|
return { hits: parsedHits.slice(0, FIND_SYMBOL_LIMIT), truncated, grepFailed: false };
|
|
669
883
|
}
|
|
670
|
-
listFiles(
|
|
884
|
+
async listFiles(relPath, depth, pattern) {
|
|
671
885
|
const t0 = Date.now();
|
|
672
|
-
this.logger.debug('SearcherService: listFiles', {
|
|
673
|
-
const
|
|
674
|
-
const
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
return { tree: '', error: `Path not found: ${relativePath}` };
|
|
886
|
+
this.logger.debug('SearcherService: listFiles', { relPath, depth, pattern });
|
|
887
|
+
const absPath = this.toAbs(relPath);
|
|
888
|
+
const loc = await this.checkRealLocation(relPath, absPath, 'dir');
|
|
889
|
+
if (!loc.ok)
|
|
890
|
+
return loc;
|
|
891
|
+
let stat;
|
|
892
|
+
try {
|
|
893
|
+
stat = await this.fs.stat(absPath);
|
|
681
894
|
}
|
|
682
|
-
|
|
683
|
-
|
|
895
|
+
catch (err) {
|
|
896
|
+
if (errorCode(err) !== 'ENOENT')
|
|
897
|
+
return { ok: false, reason: 'unreadable', code: errorCode(err) };
|
|
898
|
+
return { ok: false, reason: 'not_found', candidates: await this.candidatesOrNone(relPath, 'dir') };
|
|
684
899
|
}
|
|
900
|
+
if (!stat.isDirectory())
|
|
901
|
+
return { ok: false, reason: 'not_a_directory' };
|
|
685
902
|
const clampedDepth = Math.min(5, Math.max(1, depth));
|
|
686
903
|
const lines = [];
|
|
687
|
-
|
|
904
|
+
try {
|
|
905
|
+
await this.walkDir(absPath, 0, clampedDepth - 1, pattern, lines);
|
|
906
|
+
}
|
|
907
|
+
catch (err) {
|
|
908
|
+
return { ok: false, reason: 'unreadable', code: errorCode(err) };
|
|
909
|
+
}
|
|
688
910
|
this.logger.debug('SearcherService: listFiles result', { entries: lines.length, ms: Date.now() - t0 });
|
|
689
911
|
if (lines.length === 0) {
|
|
690
|
-
return { tree: pattern ? `No files matching "${pattern}" found.` : '(empty directory)' };
|
|
912
|
+
return { ok: true, tree: pattern ? `No files matching "${pattern}" found.` : '(empty directory)' };
|
|
691
913
|
}
|
|
692
|
-
return { tree: lines.join('\n') };
|
|
914
|
+
return { ok: true, tree: lines.join('\n') };
|
|
693
915
|
}
|
|
694
|
-
async
|
|
695
|
-
const
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
916
|
+
async openableFile(relPath, absPath) {
|
|
917
|
+
const loc = await this.checkRealLocation(relPath, absPath, 'file');
|
|
918
|
+
if (!loc.ok)
|
|
919
|
+
return loc;
|
|
920
|
+
let stat;
|
|
921
|
+
try {
|
|
922
|
+
stat = await this.fs.stat(absPath);
|
|
923
|
+
}
|
|
924
|
+
catch (err) {
|
|
925
|
+
if (errorCode(err) !== 'ENOENT')
|
|
926
|
+
return { ok: false, reason: 'unreadable', code: errorCode(err) };
|
|
927
|
+
if (await this.isIndexedPath(relPath))
|
|
928
|
+
return { ok: false, reason: 'deleted' };
|
|
929
|
+
return { ok: false, reason: 'not_found', candidates: await this.candidatesOrNone(relPath, 'file') };
|
|
930
|
+
}
|
|
931
|
+
return stat.isFile() ? null : { ok: false, reason: 'not_a_file' };
|
|
932
|
+
}
|
|
933
|
+
async readLines(absPath, onLine) {
|
|
934
|
+
const stream = this.fs.createReadStream(absPath);
|
|
706
935
|
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
707
936
|
let lineNo = 0;
|
|
708
|
-
let hasMore = false;
|
|
709
937
|
try {
|
|
710
938
|
for await (const line of rl) {
|
|
711
939
|
lineNo++;
|
|
712
|
-
if (lineNo
|
|
713
|
-
lines.push(line);
|
|
714
|
-
}
|
|
715
|
-
else {
|
|
716
|
-
hasMore = true;
|
|
717
|
-
rl.close();
|
|
718
|
-
stream.destroy();
|
|
940
|
+
if (!onLine(line, lineNo))
|
|
719
941
|
break;
|
|
720
|
-
}
|
|
721
942
|
}
|
|
943
|
+
return { ok: true, lineCount: lineNo };
|
|
722
944
|
}
|
|
723
945
|
catch (err) {
|
|
946
|
+
return { ok: false, code: errorCode(err) };
|
|
947
|
+
}
|
|
948
|
+
finally {
|
|
724
949
|
rl.close();
|
|
725
|
-
stream.destroy();
|
|
726
|
-
throw err;
|
|
950
|
+
stream.destroy?.();
|
|
727
951
|
}
|
|
728
|
-
|
|
729
|
-
|
|
952
|
+
}
|
|
953
|
+
async readFile(relPath, maxLines) {
|
|
954
|
+
const t0 = Date.now();
|
|
955
|
+
this.logger.debug('SearcherService: readFile', { relPath, maxLines });
|
|
956
|
+
const absPath = this.toAbs(relPath);
|
|
957
|
+
const failure = await this.openableFile(relPath, absPath);
|
|
958
|
+
if (failure)
|
|
959
|
+
return failure;
|
|
960
|
+
const lines = [];
|
|
961
|
+
let hasMore = false;
|
|
962
|
+
const read = await this.readLines(absPath, line => {
|
|
963
|
+
if (lines.length < maxLines) {
|
|
964
|
+
lines.push(line);
|
|
965
|
+
return true;
|
|
966
|
+
}
|
|
967
|
+
hasMore = true;
|
|
968
|
+
return false;
|
|
969
|
+
});
|
|
970
|
+
if (!read.ok)
|
|
971
|
+
return { ok: false, reason: 'unreadable', code: read.code };
|
|
972
|
+
this.logger.debug('SearcherService: readFile result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
|
|
730
973
|
return { ok: true, filePath: relPath, startLine: 1, endLine: lines.length, lines, hasMore };
|
|
731
974
|
}
|
|
732
|
-
async readFileRange(
|
|
975
|
+
async readFileRange(relPath, startLine, endLine) {
|
|
733
976
|
const t0 = Date.now();
|
|
734
|
-
this.logger.debug('SearcherService: readFileRange', {
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
if (!exists)
|
|
742
|
-
return { ok: false, reason: 'not_indexed' };
|
|
743
|
-
if (!fs.existsSync(absPath))
|
|
744
|
-
return { ok: false, reason: 'not_on_disk' };
|
|
977
|
+
this.logger.debug('SearcherService: readFileRange', { relPath, startLine, endLine });
|
|
978
|
+
if (endLine < startLine)
|
|
979
|
+
return { ok: false, reason: 'bad_range', startLine, endLine };
|
|
980
|
+
const absPath = this.toAbs(relPath);
|
|
981
|
+
const failure = await this.openableFile(relPath, absPath);
|
|
982
|
+
if (failure)
|
|
983
|
+
return failure;
|
|
745
984
|
const lines = [];
|
|
746
|
-
const stream = fs.createReadStream(absPath, { encoding: 'utf-8' });
|
|
747
|
-
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
748
|
-
let lineNo = 0;
|
|
749
985
|
let lastRead = 0;
|
|
750
986
|
let hasMore = false;
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
lines.push(line);
|
|
756
|
-
if (lineNo <= endLine)
|
|
757
|
-
lastRead = lineNo;
|
|
758
|
-
if (lineNo > endLine) {
|
|
759
|
-
hasMore = true;
|
|
760
|
-
rl.close();
|
|
761
|
-
stream.destroy();
|
|
762
|
-
break;
|
|
763
|
-
}
|
|
987
|
+
const read = await this.readLines(absPath, (line, lineNo) => {
|
|
988
|
+
if (lineNo > endLine) {
|
|
989
|
+
hasMore = true;
|
|
990
|
+
return false;
|
|
764
991
|
}
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
992
|
+
if (lineNo >= startLine)
|
|
993
|
+
lines.push(line);
|
|
994
|
+
lastRead = lineNo;
|
|
995
|
+
return true;
|
|
996
|
+
});
|
|
997
|
+
if (!read.ok)
|
|
998
|
+
return { ok: false, reason: 'unreadable', code: read.code };
|
|
999
|
+
if (!hasMore && startLine > Math.max(read.lineCount, 1)) {
|
|
1000
|
+
return { ok: false, reason: 'past_end', lineCount: read.lineCount, startLine };
|
|
770
1001
|
}
|
|
771
1002
|
this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
|
|
772
|
-
return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
|
|
1003
|
+
return { ok: true, filePath: relPath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
|
|
773
1004
|
}
|
|
774
|
-
walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
|
|
775
|
-
|
|
776
|
-
try {
|
|
777
|
-
entries = fs.readdirSync(absDir, { withFileTypes: true });
|
|
778
|
-
}
|
|
779
|
-
catch {
|
|
780
|
-
return;
|
|
781
|
-
}
|
|
1005
|
+
async walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
|
|
1006
|
+
const entries = await this.fs.readdir(absDir);
|
|
782
1007
|
entries.sort((a, b) => {
|
|
783
1008
|
const ad = a.isDirectory() ? 0 : 1;
|
|
784
1009
|
const bd = b.isDirectory() ? 0 : 1;
|
|
@@ -791,9 +1016,14 @@ class SearcherService {
|
|
|
791
1016
|
continue;
|
|
792
1017
|
const indent = ' '.repeat(currentDepth);
|
|
793
1018
|
if (entry.isDirectory()) {
|
|
794
|
-
lines.push(`${indent}${entry.name}/`);
|
|
1019
|
+
const at = lines.push(`${indent}${entry.name}/`) - 1;
|
|
795
1020
|
if (currentDepth < maxDepth) {
|
|
796
|
-
|
|
1021
|
+
try {
|
|
1022
|
+
await this.walkDir(path.join(absDir, entry.name), currentDepth + 1, maxDepth, pattern, lines);
|
|
1023
|
+
}
|
|
1024
|
+
catch {
|
|
1025
|
+
lines[at] += ' (unreadable)';
|
|
1026
|
+
}
|
|
797
1027
|
}
|
|
798
1028
|
}
|
|
799
1029
|
else {
|
|
@@ -803,17 +1033,14 @@ class SearcherService {
|
|
|
803
1033
|
}
|
|
804
1034
|
}
|
|
805
1035
|
}
|
|
806
|
-
/**
|
|
807
|
-
* ERE pattern for a grep-fallback kind, or null when that kind is never
|
|
808
|
-
* grepped (`function_property`/`method_signature` are index-only; a bare
|
|
809
|
-
* `constructor` search only makes sense when the name itself is `constructor`).
|
|
810
|
-
*/
|
|
811
1036
|
buildFindSymbolPattern(name, kind) {
|
|
812
1037
|
if (kind === 'function_property' || kind === 'method_signature')
|
|
813
1038
|
return null;
|
|
1039
|
+
if (GRAPHQL_KINDS.has(kind))
|
|
1040
|
+
return null;
|
|
814
1041
|
if (kind === 'constructor')
|
|
815
1042
|
return name === 'constructor' ? `(^|[[:space:]])constructor[[:space:]]*\\(` : null;
|
|
816
|
-
const esc = name.replace(/\$/g, '\\$');
|
|
1043
|
+
const esc = name.replace(/\$/g, '\\$');
|
|
817
1044
|
const MODIFIER = '(public|private|protected|static|abstract|async|override)';
|
|
818
1045
|
const BOUNDARY = '(^|[^[:alnum:]_$])';
|
|
819
1046
|
const patterns = {
|
|
@@ -830,7 +1057,6 @@ class SearcherService {
|
|
|
830
1057
|
return Object.values(patterns).join('|');
|
|
831
1058
|
return patterns[kind];
|
|
832
1059
|
}
|
|
833
|
-
/** 1-based index of the first whole-word occurrence of `name` in `line`, or -1. */
|
|
834
1060
|
static wholeWordIndex(line, name) {
|
|
835
1061
|
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
836
1062
|
const re = new RegExp(`(?<![\\w$#])${escaped}(?![\\w$#])`);
|
|
@@ -860,11 +1086,6 @@ class SearcherService {
|
|
|
860
1086
|
return 'method';
|
|
861
1087
|
return 'variable';
|
|
862
1088
|
}
|
|
863
|
-
/**
|
|
864
|
-
* Async wrapper around grep to avoid blocking the event loop. `ok: false`
|
|
865
|
-
* means grep could not be spawned or exited with status ≥ 2 (distinguishable
|
|
866
|
-
* from a clean "no matches", which is `ok: true, output: null`).
|
|
867
|
-
*/
|
|
868
1089
|
spawnGrep(args) {
|
|
869
1090
|
return new Promise((resolve) => {
|
|
870
1091
|
const proc = (0, child_process_1.spawn)('grep', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
@@ -879,17 +1100,17 @@ class SearcherService {
|
|
|
879
1100
|
});
|
|
880
1101
|
proc.stderr.on('data', () => { });
|
|
881
1102
|
proc.on('error', (err) => {
|
|
882
|
-
this.logger.error('SearcherService: grep spawn failed', {
|
|
883
|
-
resolve({ ok: false });
|
|
1103
|
+
this.logger.error('SearcherService: grep spawn failed', { err });
|
|
1104
|
+
resolve({ ok: false, output: null });
|
|
884
1105
|
});
|
|
885
1106
|
proc.on('close', (code) => {
|
|
886
|
-
|
|
1107
|
+
const output = chunks.length === 0 ? null : Buffer.concat(chunks).toString('utf-8');
|
|
887
1108
|
if (code !== null && code >= 2) {
|
|
888
1109
|
this.logger.warn('SearcherService: grep exited with code', { code });
|
|
889
|
-
resolve({ ok: false });
|
|
1110
|
+
resolve({ ok: false, output });
|
|
890
1111
|
return;
|
|
891
1112
|
}
|
|
892
|
-
resolve({ ok: true, output
|
|
1113
|
+
resolve({ ok: true, output });
|
|
893
1114
|
});
|
|
894
1115
|
});
|
|
895
1116
|
}
|