codesearch-mcp 0.0.17-alpha.1 → 0.1.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +95 -6
- package/dist/app.js +70 -10
- package/dist/src/commands/ConfigCommand.d.ts +1 -0
- package/dist/src/commands/ConfigCommand.js +84 -4
- package/dist/src/commands/IndexCommand.js +35 -7
- package/dist/src/commands/McpCommand.js +9 -4
- package/dist/src/commands/{SetupCommand.d.ts → WebCommand.d.ts} +3 -2
- package/dist/src/commands/WebCommand.js +56 -0
- package/dist/src/config-store.d.ts +85 -4
- package/dist/src/config-store.js +157 -32
- package/dist/src/config.d.ts +32 -8
- package/dist/src/config.js +93 -30
- package/dist/src/constants.d.ts +35 -0
- package/dist/src/constants.js +57 -4
- package/dist/src/container.d.ts +10 -6
- package/dist/src/container.js +10 -5
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +111 -0
- package/dist/src/db/schema.js +15 -1
- package/dist/src/graph.d.ts +60 -0
- package/dist/src/graph.js +239 -120
- package/dist/src/help.js +20 -5
- package/dist/src/interfaces/IAstParser.d.ts +20 -3
- package/dist/src/interfaces/IChunker.d.ts +24 -0
- package/dist/src/interfaces/IChunker.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +27 -0
- package/dist/src/interfaces/IDatabase.js +2 -0
- package/dist/src/interfaces/IEmbedder.d.ts +15 -2
- package/dist/src/interfaces/ILogger.d.ts +10 -0
- package/dist/src/interfaces/ILogger.js +2 -0
- package/dist/src/interfaces/IMcpToolHandler.d.ts +22 -0
- package/dist/src/interfaces/IMcpToolHandler.js +2 -0
- package/dist/src/interfaces/IMonorepoResolver.d.ts +33 -0
- package/dist/src/interfaces/IMonorepoResolver.js +2 -0
- package/dist/src/interfaces/IProjectDiscovery.d.ts +28 -0
- package/dist/src/interfaces/IProjectDiscovery.js +2 -0
- package/dist/src/interfaces/ISearcher.d.ts +37 -0
- package/dist/src/interfaces/ISearcher.js +2 -0
- package/dist/src/interfaces/index.d.ts +8 -0
- package/dist/src/interfaces/index.js +2 -0
- package/dist/src/owner-path.d.ts +10 -0
- package/dist/src/owner-path.js +27 -0
- package/dist/src/project-selector.d.ts +1 -1
- package/dist/src/services/AstChunker.d.ts +31 -0
- package/dist/src/services/AstChunker.js +114 -0
- package/dist/src/services/AstParserService.d.ts +10 -6
- package/dist/src/services/AstParserService.js +60 -15
- package/dist/src/services/DatabaseService.d.ts +14 -1
- package/dist/src/services/DatabaseService.js +70 -25
- package/dist/src/services/EmbedderFactory.d.ts +9 -0
- package/dist/src/services/EmbedderFactory.js +29 -0
- package/dist/src/services/IndexerService.d.ts +22 -12
- package/dist/src/services/IndexerService.js +106 -46
- package/dist/src/services/InfinityEmbedder.d.ts +79 -0
- package/dist/src/services/InfinityEmbedder.js +199 -0
- package/dist/src/services/LoggerService.d.ts +2 -1
- package/dist/src/services/McpServer.d.ts +16 -7
- package/dist/src/services/McpServer.js +54 -367
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +62 -0
- package/dist/src/services/MonorepoAwareDiscoveryService.js +125 -0
- package/dist/src/services/NxMonorepoResolver.d.ts +45 -0
- package/dist/src/services/NxMonorepoResolver.js +308 -0
- package/dist/src/services/OllamaEmbedder.d.ts +19 -5
- package/dist/src/services/OllamaEmbedder.js +91 -36
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
- package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
- package/dist/src/services/ProjectDiscoveryService.d.ts +8 -0
- package/dist/src/{env-loader.js → services/ProjectDiscoveryService.js} +16 -5
- package/dist/src/services/SearcherService.d.ts +46 -7
- package/dist/src/services/SearcherService.js +183 -53
- package/dist/src/services/VoyageEmbedder.d.ts +22 -0
- package/dist/src/services/VoyageEmbedder.js +72 -0
- package/dist/src/services/WebServerService.d.ts +77 -0
- package/dist/src/services/WebServerService.js +267 -0
- package/dist/src/services/http-retry.d.ts +19 -0
- package/dist/src/services/http-retry.js +54 -0
- package/dist/src/services/reranker.d.ts +15 -0
- package/dist/src/services/reranker.js +44 -0
- package/dist/src/setup.d.ts +6 -0
- package/dist/src/setup.js +249 -31
- package/dist/src/tools/FindSymbolTool.d.ts +8 -0
- package/dist/src/tools/FindSymbolTool.js +45 -0
- package/dist/src/tools/GetCallersTool.d.ts +8 -0
- package/dist/src/tools/GetCallersTool.js +42 -0
- package/dist/src/tools/GetDependenciesTool.d.ts +8 -0
- package/dist/src/tools/GetDependenciesTool.js +38 -0
- package/dist/src/tools/GetDependentsTool.d.ts +8 -0
- package/dist/src/tools/GetDependentsTool.js +57 -0
- package/dist/src/tools/ListFilesTool.d.ts +8 -0
- package/dist/src/tools/ListFilesTool.js +39 -0
- package/dist/src/tools/ListProjectsTool.d.ts +8 -0
- package/dist/src/tools/ListProjectsTool.js +44 -0
- package/dist/src/tools/ReadFileRangeTool.d.ts +8 -0
- package/dist/src/tools/ReadFileRangeTool.js +44 -0
- package/dist/src/tools/ReadFileTool.d.ts +8 -0
- package/dist/src/tools/ReadFileTool.js +42 -0
- package/dist/src/tools/SearchCodebaseTool.d.ts +13 -0
- package/dist/src/tools/SearchCodebaseTool.js +81 -0
- package/dist/src/tools/format-helpers.d.ts +8 -0
- package/dist/src/tools/format-helpers.js +58 -0
- package/dist/src/tools/index.d.ts +9 -0
- package/dist/src/tools/index.js +21 -0
- package/dist/src/web/dashboard.d.ts +9 -0
- package/dist/src/web/dashboard.js +342 -0
- package/docker/docker-compose.infinity-gpu.yml +19 -0
- package/docker/docker-compose.yml +25 -0
- package/docs/web-dashboard.md +187 -0
- package/package.json +25 -3
- package/dist/src/ast.d.ts +0 -2
- package/dist/src/ast.js +0 -110
- package/dist/src/commands/SetupCommand.js +0 -14
- package/dist/src/db/data-source.d.ts +0 -2
- package/dist/src/db/data-source.js +0 -13
- package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
- package/dist/src/db/entities/code-chunk.entity.js +0 -54
- package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
- package/dist/src/db/entities/code-dependency.entity.js +0 -54
- package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
- package/dist/src/db/entities/code-symbol.entity.js +0 -59
- package/dist/src/db/init.d.ts +0 -2
- package/dist/src/db/init.js +0 -62
- package/dist/src/env-loader.d.ts +0 -1
- package/dist/src/indexer.d.ts +0 -17
- package/dist/src/indexer.js +0 -242
- package/dist/src/mcp.d.ts +0 -1
- package/dist/src/mcp.js +0 -403
- package/dist/src/ollama.d.ts +0 -2
- package/dist/src/ollama.js +0 -6
- package/dist/src/searcher.d.ts +0 -74
- package/dist/src/searcher.js +0 -446
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -9
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
export interface EmbedderDeps {
|
|
5
|
+
fetchImpl?: typeof fetch;
|
|
6
|
+
baseDelayMs?: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Embedder for any server exposing an OpenAI-style `/embeddings` endpoint:
|
|
10
|
+
* OpenAI cloud, Infinity, TEI, vLLM, LM Studio, LocalAI. One request carries all
|
|
11
|
+
* inputs (`{ input: string[], model, dimensions? }`) and returns
|
|
12
|
+
* `{ data: [{ embedding, index }] }`, re-ordered to input order.
|
|
13
|
+
*/
|
|
14
|
+
export declare class OpenAICompatibleEmbedder implements IEmbedder {
|
|
15
|
+
protected config: Config;
|
|
16
|
+
private logger;
|
|
17
|
+
readonly dimensions: number;
|
|
18
|
+
rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
|
|
19
|
+
private readonly url;
|
|
20
|
+
private readonly doFetch;
|
|
21
|
+
private readonly baseDelayMs?;
|
|
22
|
+
constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
|
|
23
|
+
embed(text: string, inputType?: InputType): Promise<number[]>;
|
|
24
|
+
embedBatch(texts: string[], _inputType?: InputType): Promise<number[][]>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.OpenAICompatibleEmbedder = void 0;
|
|
4
|
+
const LoggerService_1 = require("./LoggerService");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const reranker_1 = require("./reranker");
|
|
7
|
+
const http_retry_1 = require("./http-retry");
|
|
8
|
+
const constants_1 = require("../constants");
|
|
9
|
+
/**
|
|
10
|
+
* Embedder for any server exposing an OpenAI-style `/embeddings` endpoint:
|
|
11
|
+
* OpenAI cloud, Infinity, TEI, vLLM, LM Studio, LocalAI. One request carries all
|
|
12
|
+
* inputs (`{ input: string[], model, dimensions? }`) and returns
|
|
13
|
+
* `{ data: [{ embedding, index }] }`, re-ordered to input order.
|
|
14
|
+
*/
|
|
15
|
+
class OpenAICompatibleEmbedder {
|
|
16
|
+
config;
|
|
17
|
+
logger;
|
|
18
|
+
dimensions;
|
|
19
|
+
rerank;
|
|
20
|
+
url;
|
|
21
|
+
doFetch;
|
|
22
|
+
baseDelayMs;
|
|
23
|
+
constructor(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.logger = logger;
|
|
26
|
+
const base = config.embedding.baseURL.replace(/\/+$/, '');
|
|
27
|
+
// Allow a baseURL that already includes the path (…/v1) or not.
|
|
28
|
+
this.url = /\/embeddings$/.test(base) ? base : `${base}/embeddings`;
|
|
29
|
+
this.doFetch = deps.fetchImpl ?? fetch;
|
|
30
|
+
this.baseDelayMs = deps.baseDelayMs;
|
|
31
|
+
this.dimensions = config.embedding.dimensions ?? (0, constants_1.registryDimension)(config.embedding.model) ?? 0;
|
|
32
|
+
this.rerank = (0, reranker_1.makeReranker)(config, logger, deps);
|
|
33
|
+
}
|
|
34
|
+
async embed(text, inputType) {
|
|
35
|
+
const [v] = await this.embedBatch([text], inputType);
|
|
36
|
+
return v;
|
|
37
|
+
}
|
|
38
|
+
async embedBatch(texts, _inputType) {
|
|
39
|
+
if (texts.length === 0)
|
|
40
|
+
return [];
|
|
41
|
+
return (0, http_retry_1.withRetry)(async () => {
|
|
42
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
43
|
+
// Resolve the key lazily — an unset `env:` ref throws here (request time),
|
|
44
|
+
// not at construction, so `rag-web`/`rag-mcp` still start.
|
|
45
|
+
const apiKey = (0, config_1.resolveEnvRef)(this.config.embedding.apiKey);
|
|
46
|
+
if (apiKey)
|
|
47
|
+
headers['Authorization'] = `Bearer ${apiKey}`;
|
|
48
|
+
const body = { model: this.config.embedding.model, input: texts };
|
|
49
|
+
if (this.config.embedding.dimensions != null)
|
|
50
|
+
body.dimensions = this.config.embedding.dimensions;
|
|
51
|
+
const res = await this.doFetch(this.url, {
|
|
52
|
+
method: 'POST',
|
|
53
|
+
headers,
|
|
54
|
+
signal: AbortSignal.timeout(120_000),
|
|
55
|
+
body: JSON.stringify(body),
|
|
56
|
+
});
|
|
57
|
+
if (!res.ok) {
|
|
58
|
+
const errBody = (await res.text().catch(() => res.statusText)).slice(0, 200);
|
|
59
|
+
throw new http_retry_1.HttpError(res.status, `embeddings request failed: ${res.status} ${errBody}`);
|
|
60
|
+
}
|
|
61
|
+
const data = await res.json();
|
|
62
|
+
// Guard against a partial/mismatched response — a missing vector would
|
|
63
|
+
// otherwise be inserted as "[undefined]" and fail the whole file, or (with
|
|
64
|
+
// reordering + no index) silently misalign vectors to the wrong chunks.
|
|
65
|
+
if (!Array.isArray(data.data) || data.data.length !== texts.length) {
|
|
66
|
+
throw new Error(`embeddings response returned ${data.data?.length ?? 0} vectors for ${texts.length} inputs`);
|
|
67
|
+
}
|
|
68
|
+
// Sort by `index` so vectors line up with input order regardless of server ordering.
|
|
69
|
+
return [...data.data]
|
|
70
|
+
.sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
|
|
71
|
+
.map(d => d.embedding);
|
|
72
|
+
}, { logger: this.logger, baseDelayMs: this.baseDelayMs, label: 'openai-embed' });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.OpenAICompatibleEmbedder = OpenAICompatibleEmbedder;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { IProjectDiscovery, DiscoveredProject } from '../interfaces/IProjectDiscovery';
|
|
2
|
+
/**
|
|
3
|
+
* Default project discovery: finds immediate subdirectories
|
|
4
|
+
* of the workspace root that contain a package.json.
|
|
5
|
+
*/
|
|
6
|
+
export declare class ProjectDiscoveryService implements IProjectDiscovery {
|
|
7
|
+
discover(rootDir: string): DiscoveredProject[];
|
|
8
|
+
}
|
|
@@ -33,9 +33,20 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
|
|
36
|
+
exports.ProjectDiscoveryService = void 0;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
37
38
|
const path = __importStar(require("path"));
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
39
|
+
const file_utils_1 = require("../file-utils");
|
|
40
|
+
/**
|
|
41
|
+
* Default project discovery: finds immediate subdirectories
|
|
42
|
+
* of the workspace root that contain a package.json.
|
|
43
|
+
*/
|
|
44
|
+
class ProjectDiscoveryService {
|
|
45
|
+
discover(rootDir) {
|
|
46
|
+
return fs.readdirSync(rootDir, { withFileTypes: true })
|
|
47
|
+
.filter(d => d.isDirectory() && !file_utils_1.IGNORE_DIRS.has(d.name))
|
|
48
|
+
.map(d => ({ name: d.name, path: path.join(rootDir, d.name) }))
|
|
49
|
+
.filter(p => fs.existsSync(path.join(p.path, 'package.json')));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
exports.ProjectDiscoveryService = ProjectDiscoveryService;
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import type { Config } from '../config';
|
|
2
|
-
import type { IEmbedder } from '../interfaces
|
|
3
|
-
import type {
|
|
2
|
+
import type { IEmbedder } from '../interfaces';
|
|
3
|
+
import type { IDatabase } from '../interfaces';
|
|
4
4
|
import { type ProjectMeta } from '../file-utils';
|
|
5
|
-
import {
|
|
5
|
+
import type { ILogger } from '../interfaces';
|
|
6
|
+
import type { IProjectDiscovery } from '../interfaces';
|
|
6
7
|
export interface ChunkRow {
|
|
7
8
|
serviceName: string;
|
|
8
9
|
filePath: string;
|
|
9
10
|
startLine: number;
|
|
10
11
|
content: string;
|
|
12
|
+
ownerPath?: string | null;
|
|
11
13
|
}
|
|
12
14
|
export interface SearchResult extends ChunkRow {
|
|
13
15
|
score: number;
|
|
@@ -44,6 +46,10 @@ export interface DependencyRow {
|
|
|
44
46
|
export interface ProjectRow extends ProjectMeta {
|
|
45
47
|
serviceName: string;
|
|
46
48
|
chunks: number;
|
|
49
|
+
packageName?: string | null;
|
|
50
|
+
type?: 'app' | 'lib';
|
|
51
|
+
/** Real filesystem root of the project (resolved via discovery). */
|
|
52
|
+
path?: string;
|
|
47
53
|
}
|
|
48
54
|
export interface GrepMatch {
|
|
49
55
|
filePath: string;
|
|
@@ -67,13 +73,42 @@ export interface SymbolHit {
|
|
|
67
73
|
export declare class SearcherService {
|
|
68
74
|
private db;
|
|
69
75
|
private embedder;
|
|
76
|
+
private discovery;
|
|
70
77
|
private config;
|
|
71
78
|
private logger;
|
|
72
|
-
constructor(db:
|
|
79
|
+
constructor(db: IDatabase, embedder: IEmbedder, discovery: IProjectDiscovery, config: Config, logger?: ILogger);
|
|
80
|
+
/** Lazily-built, cached map of project name → real filesystem root. */
|
|
81
|
+
private rootMap?;
|
|
82
|
+
/** Cached, lazily-built map of leaf project name → real filesystem root (via discovery). */
|
|
83
|
+
private projectRootMap;
|
|
84
|
+
/**
|
|
85
|
+
* Resolve a project name to its real filesystem root. Uses discovery (leaf
|
|
86
|
+
* projects) so nested-monorepo names like `repo/app` map to their true path;
|
|
87
|
+
* falls back to `join(workspaceRoot, name)` when the name is unknown (e.g. a
|
|
88
|
+
* project since removed from disk, or a flat project not surfaced by discovery).
|
|
89
|
+
*/
|
|
90
|
+
private resolveProjectRoot;
|
|
91
|
+
/**
|
|
92
|
+
* Project-level dependency edges aggregated from `code_dependency`: an edge
|
|
93
|
+
* `A → B` means some file in project A imports a file owned by project B.
|
|
94
|
+
* Used by the graph to draw links between a monorepo's sub-projects. Only
|
|
95
|
+
* edges whose target file belongs to an indexed project are returned (targets
|
|
96
|
+
* outside any indexed project — e.g. unindexed libs — are skipped).
|
|
97
|
+
*/
|
|
98
|
+
getProjectEdges(): Promise<{
|
|
99
|
+
from: string;
|
|
100
|
+
to: string;
|
|
101
|
+
}[]>;
|
|
73
102
|
prepareQuery(raw: string): Promise<{
|
|
74
103
|
query: string;
|
|
75
104
|
translated: boolean;
|
|
76
105
|
}>;
|
|
106
|
+
/**
|
|
107
|
+
* Refuse when the configured embedding model disagrees with the one that built
|
|
108
|
+
* the index — embedding a query into a foreign vector space returns noise. A
|
|
109
|
+
* missing `index_meta` (un-migrated DB / not yet indexed) does not block.
|
|
110
|
+
*/
|
|
111
|
+
assertIndexMatches(): Promise<void>;
|
|
77
112
|
listProjects(): Promise<ProjectRow[]>;
|
|
78
113
|
searchChunks(query: string, limit: number, project: string | null, threshold?: number): Promise<SearchResult[]>;
|
|
79
114
|
fetchNeighbors(results: SearchResult[]): Promise<NeighborResult[]>;
|
|
@@ -81,10 +116,10 @@ export declare class SearcherService {
|
|
|
81
116
|
getDependencies(filePath: string, project: string, symbol?: string | null, transitive?: boolean): Promise<DependencyRow[]>;
|
|
82
117
|
getDependents(filePath: string, project: string, symbol?: string | null, transitive?: boolean): Promise<DependencyRow[]>;
|
|
83
118
|
getCallers(symbol: string, project: string, calleeFile?: string | null): Promise<CallerRow[]>;
|
|
84
|
-
grepSymbol(symbol: string, project: string, excludeFile: string): {
|
|
119
|
+
grepSymbol(symbol: string, project: string, excludeFile: string): Promise<{
|
|
85
120
|
matches: GrepMatch[];
|
|
86
121
|
truncated: boolean;
|
|
87
|
-
}
|
|
122
|
+
}>;
|
|
88
123
|
findSymbol(name: string, project: string | null, kind: SymbolKind): Promise<{
|
|
89
124
|
hits: SymbolHit[];
|
|
90
125
|
truncated: boolean;
|
|
@@ -97,8 +132,12 @@ export declare class SearcherService {
|
|
|
97
132
|
};
|
|
98
133
|
readFileAbsolute(absPath: string, maxLines: number): Promise<FileRangeResult>;
|
|
99
134
|
readFileRange(filePath: string, startLine: number, endLine: number): Promise<FileRangeResult>;
|
|
100
|
-
private isNonEnglish;
|
|
101
135
|
private walkDir;
|
|
102
136
|
private buildFindSymbolPattern;
|
|
103
137
|
private inferKind;
|
|
138
|
+
/**
|
|
139
|
+
* Async wrapper around grep to avoid blocking the event loop.
|
|
140
|
+
* Returns stdout as a string, or null if grep returned no results or errored.
|
|
141
|
+
*/
|
|
142
|
+
private spawnGrep;
|
|
104
143
|
}
|
|
@@ -39,7 +39,13 @@ const path = __importStar(require("path"));
|
|
|
39
39
|
const readline = __importStar(require("readline"));
|
|
40
40
|
const child_process_1 = require("child_process");
|
|
41
41
|
const file_utils_1 = require("../file-utils");
|
|
42
|
-
const
|
|
42
|
+
const MonorepoAwareDiscoveryService_1 = require("./MonorepoAwareDiscoveryService");
|
|
43
|
+
const SILENT_LOGGER = {
|
|
44
|
+
debug() { },
|
|
45
|
+
info() { },
|
|
46
|
+
warn() { },
|
|
47
|
+
error() { },
|
|
48
|
+
};
|
|
43
49
|
const EMPTY_META = { description: null, stack: null, notes: null, relations: null };
|
|
44
50
|
const GREP_MAX_RESULTS = 200;
|
|
45
51
|
const GREP_RAW_LIMIT = GREP_MAX_RESULTS * 5;
|
|
@@ -49,51 +55,138 @@ const FIND_SYMBOL_RAW_LIMIT = FIND_SYMBOL_LIMIT * 10;
|
|
|
49
55
|
class SearcherService {
|
|
50
56
|
db;
|
|
51
57
|
embedder;
|
|
58
|
+
discovery;
|
|
52
59
|
config;
|
|
53
60
|
logger;
|
|
54
|
-
constructor(db, embedder, config, logger =
|
|
61
|
+
constructor(db, embedder, discovery, config, logger = SILENT_LOGGER) {
|
|
55
62
|
this.db = db;
|
|
56
63
|
this.embedder = embedder;
|
|
64
|
+
this.discovery = discovery;
|
|
57
65
|
this.config = config;
|
|
58
66
|
this.logger = logger;
|
|
59
67
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
68
|
+
/** Lazily-built, cached map of project name → real filesystem root. */
|
|
69
|
+
rootMap;
|
|
70
|
+
/** Cached, lazily-built map of leaf project name → real filesystem root (via discovery). */
|
|
71
|
+
projectRootMap() {
|
|
72
|
+
if (!this.rootMap) {
|
|
73
|
+
this.rootMap = new Map((0, MonorepoAwareDiscoveryService_1.toLeafProjects)(this.discovery.discover(this.config.workspaceRoot)).map(p => [p.name, p.path]));
|
|
74
|
+
}
|
|
75
|
+
return this.rootMap;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Resolve a project name to its real filesystem root. Uses discovery (leaf
|
|
79
|
+
* projects) so nested-monorepo names like `repo/app` map to their true path;
|
|
80
|
+
* falls back to `join(workspaceRoot, name)` when the name is unknown (e.g. a
|
|
81
|
+
* project since removed from disk, or a flat project not surfaced by discovery).
|
|
82
|
+
*/
|
|
83
|
+
resolveProjectRoot(name) {
|
|
84
|
+
let root = this.projectRootMap().get(name);
|
|
85
|
+
if (root === undefined) {
|
|
86
|
+
// Cache miss — the project may have been added/renamed on disk since the
|
|
87
|
+
// map was built (mcp/web servers are long-lived). Rebuild once and retry
|
|
88
|
+
// before the flat fallback, which is wrong for nested-monorepo names
|
|
89
|
+
// (`repo/app` lives at `repo/apps/app`, not `repo/app`).
|
|
90
|
+
this.rootMap = undefined;
|
|
91
|
+
root = this.projectRootMap().get(name);
|
|
92
|
+
}
|
|
93
|
+
return root ?? path.join(this.config.workspaceRoot, name);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Project-level dependency edges aggregated from `code_dependency`: an edge
|
|
97
|
+
* `A → B` means some file in project A imports a file owned by project B.
|
|
98
|
+
* Used by the graph to draw links between a monorepo's sub-projects. Only
|
|
99
|
+
* edges whose target file belongs to an indexed project are returned (targets
|
|
100
|
+
* outside any indexed project — e.g. unindexed libs — are skipped).
|
|
101
|
+
*/
|
|
102
|
+
async getProjectEdges() {
|
|
103
|
+
const rows = await this.db.query(`SELECT DISTINCT project, "toFile" FROM code_dependency`);
|
|
104
|
+
// Reverse map: workspace-relative project root → name, longest first so the
|
|
105
|
+
// most specific owner wins (e.g. repo/apps/account before repo).
|
|
106
|
+
const byRelPath = [...this.projectRootMap().entries()]
|
|
107
|
+
.map(([name, abs]) => ({ name, rel: path.relative(this.config.workspaceRoot, abs) }))
|
|
108
|
+
.sort((a, b) => b.rel.length - a.rel.length);
|
|
109
|
+
const ownerOf = (toFile) => {
|
|
110
|
+
for (const { name, rel } of byRelPath) {
|
|
111
|
+
if (toFile === rel || toFile.startsWith(rel + path.sep) || toFile.startsWith(rel + '/'))
|
|
112
|
+
return name;
|
|
113
|
+
}
|
|
114
|
+
return null;
|
|
115
|
+
};
|
|
116
|
+
const seen = new Set();
|
|
117
|
+
const edges = [];
|
|
118
|
+
for (const { project, toFile } of rows) {
|
|
119
|
+
const to = ownerOf(toFile);
|
|
120
|
+
if (!to || to === project)
|
|
121
|
+
continue;
|
|
122
|
+
const key = project + ' ' + to;
|
|
123
|
+
if (seen.has(key))
|
|
124
|
+
continue;
|
|
125
|
+
seen.add(key);
|
|
126
|
+
edges.push({ from: project, to });
|
|
66
127
|
}
|
|
67
|
-
|
|
128
|
+
return edges;
|
|
129
|
+
}
|
|
130
|
+
// Query translation was removed (legacy). The query is embedded as-is; the
|
|
131
|
+
// method is kept so the tool layer needs no change and `translated` is always
|
|
132
|
+
// false.
|
|
133
|
+
async prepareQuery(raw) {
|
|
68
134
|
return { query: raw, translated: false };
|
|
69
135
|
}
|
|
136
|
+
/**
|
|
137
|
+
* Refuse when the configured embedding model disagrees with the one that built
|
|
138
|
+
* the index — embedding a query into a foreign vector space returns noise. A
|
|
139
|
+
* missing `index_meta` (un-migrated DB / not yet indexed) does not block.
|
|
140
|
+
*/
|
|
141
|
+
async assertIndexMatches() {
|
|
142
|
+
const meta = await this.db.readIndexMeta();
|
|
143
|
+
if (!meta)
|
|
144
|
+
return;
|
|
145
|
+
const { provider, model } = this.config.embedding;
|
|
146
|
+
if (meta.provider !== provider || meta.model !== model) {
|
|
147
|
+
throw new Error(`Index was built with ${meta.provider}/${meta.model} but the config selects ${provider}/${model}. ` +
|
|
148
|
+
`Run \`rag-index --reindex\` to rebuild the index for the new model.`);
|
|
149
|
+
}
|
|
150
|
+
// Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
|
|
151
|
+
// override) would embed the query at a width the column can't hold — pgvector
|
|
152
|
+
// would throw an opaque "different vector dimensions" error mid-query. Catch
|
|
153
|
+
// it here with the friendly hint. `embedder.dimensions` is 0 when the width is
|
|
154
|
+
// only knowable by probing, in which case we skip (can't compare cheaply).
|
|
155
|
+
const dim = this.embedder.dimensions;
|
|
156
|
+
if (dim > 0 && dim !== meta.dimensions) {
|
|
157
|
+
throw new Error(`Index was built at ${meta.dimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
|
|
158
|
+
`Run \`rag-index --reindex\` to rebuild the index at the new dimension.`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
70
161
|
async listProjects() {
|
|
71
162
|
const rows = await this.db.query(`SELECT "serviceName", COUNT(*)::int AS chunks
|
|
72
163
|
FROM code_chunk
|
|
73
164
|
GROUP BY "serviceName"
|
|
74
165
|
ORDER BY "serviceName"`);
|
|
75
166
|
return rows.map(r => {
|
|
76
|
-
const
|
|
77
|
-
|
|
167
|
+
const projectRoot = this.resolveProjectRoot(r.serviceName);
|
|
168
|
+
const meta = (0, file_utils_1.readRagMeta)(projectRoot);
|
|
169
|
+
return { ...r, path: projectRoot, ...(typeof meta === 'object' ? meta : EMPTY_META) };
|
|
78
170
|
});
|
|
79
171
|
}
|
|
80
172
|
async searchChunks(query, limit, project, threshold) {
|
|
81
173
|
const t0 = Date.now();
|
|
174
|
+
await this.assertIndexMatches();
|
|
82
175
|
const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
|
|
83
176
|
const fetchLimit = this.embedder.rerank
|
|
84
|
-
? Math.min(limit * this.config.
|
|
177
|
+
? Math.min(limit * this.config.reranker.overfetch, 200)
|
|
85
178
|
: limit;
|
|
86
179
|
this.logger.debug('SearcherService: searchChunks', {
|
|
87
180
|
query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
|
|
88
181
|
});
|
|
89
|
-
const embedding = await this.embedder.embed(query);
|
|
182
|
+
const embedding = await this.embedder.embed(query, 'query');
|
|
90
183
|
const params = [`[${embedding}]`, effectiveThreshold, fetchLimit];
|
|
91
184
|
let projectFilter = '';
|
|
92
185
|
if (project) {
|
|
93
186
|
params.push(project);
|
|
94
187
|
projectFilter = `AND "serviceName" = $${params.length}`;
|
|
95
188
|
}
|
|
96
|
-
const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content,
|
|
189
|
+
const candidates = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
|
|
97
190
|
ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
|
|
98
191
|
FROM code_chunk
|
|
99
192
|
WHERE embedding <=> $1 < $2
|
|
@@ -111,11 +204,13 @@ class SearcherService {
|
|
|
111
204
|
try {
|
|
112
205
|
const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
|
|
113
206
|
this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
|
|
114
|
-
|
|
207
|
+
// Cap at `limit`: some rerank servers ignore `top_n` and echo back every
|
|
208
|
+
// candidate, so slicing here keeps the result count within the request.
|
|
209
|
+
return indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
|
|
115
210
|
}
|
|
116
211
|
catch (err) {
|
|
117
212
|
this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
|
|
118
|
-
reason: err.message,
|
|
213
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
119
214
|
ms: Date.now() - t0,
|
|
120
215
|
});
|
|
121
216
|
return candidates.slice(0, limit);
|
|
@@ -168,6 +263,8 @@ class SearcherService {
|
|
|
168
263
|
if (pairs.length === 0)
|
|
169
264
|
return new Map();
|
|
170
265
|
const n = pairs.length;
|
|
266
|
+
// Use proportional limit: 50 imports + 50 usedBy per file
|
|
267
|
+
const proportionalLimit = Math.max(2000, n * 100);
|
|
171
268
|
const firstHalf = pairs.map((_, i) => `($${i * 2 + 1}::text, $${i * 2 + 2}::text)`).join(', ');
|
|
172
269
|
const secondHalf = pairs.map((_, i) => `($${n * 2 + i * 2 + 1}::text, $${n * 2 + i * 2 + 2}::text)`).join(', ');
|
|
173
270
|
const params = [
|
|
@@ -184,7 +281,7 @@ class SearcherService {
|
|
|
184
281
|
FROM (VALUES ${secondHalf}) AS r(fp, proj)
|
|
185
282
|
JOIN code_dependency d ON d."toFile" = r.fp AND d.project = r.proj
|
|
186
283
|
|
|
187
|
-
LIMIT
|
|
284
|
+
LIMIT ${proportionalLimit}`, params);
|
|
188
285
|
const ctx = new Map();
|
|
189
286
|
for (const { filePath } of pairs)
|
|
190
287
|
ctx.set(filePath, { imports: new Set(), usedBy: new Set() });
|
|
@@ -227,7 +324,7 @@ class SearcherService {
|
|
|
227
324
|
FROM code_dependency
|
|
228
325
|
WHERE "fromFile" = $1 AND project = $2
|
|
229
326
|
|
|
230
|
-
UNION
|
|
327
|
+
UNION ALL
|
|
231
328
|
|
|
232
329
|
SELECT d."toFile", deps.depth + 1, deps."toFile" AS via
|
|
233
330
|
FROM code_dependency d
|
|
@@ -270,7 +367,7 @@ class SearcherService {
|
|
|
270
367
|
FROM code_dependency
|
|
271
368
|
WHERE "toFile" = $1 AND project = $2
|
|
272
369
|
|
|
273
|
-
UNION
|
|
370
|
+
UNION ALL
|
|
274
371
|
|
|
275
372
|
SELECT d."fromFile", rdeps.depth + 1, rdeps."fromFile" AS via
|
|
276
373
|
FROM code_dependency d
|
|
@@ -308,25 +405,23 @@ class SearcherService {
|
|
|
308
405
|
this.logger.debug('SearcherService: getCallers result', { count: rows.length, ms: Date.now() - t0 });
|
|
309
406
|
return rows;
|
|
310
407
|
}
|
|
311
|
-
grepSymbol(symbol, project, excludeFile) {
|
|
408
|
+
async grepSymbol(symbol, project, excludeFile) {
|
|
312
409
|
const t0 = Date.now();
|
|
313
410
|
this.logger.debug('SearcherService: grepSymbol', { symbol, project, excludeFile });
|
|
314
|
-
const projectDir =
|
|
315
|
-
const result =
|
|
411
|
+
const projectDir = this.resolveProjectRoot(project);
|
|
412
|
+
const result = await this.spawnGrep([
|
|
316
413
|
'-rn', '-F', '-w',
|
|
317
414
|
'-m', String(GREP_RAW_LIMIT),
|
|
318
415
|
'--include=*.ts', '--include=*.tsx', '--include=*.js', '--include=*.jsx',
|
|
319
416
|
...GREP_EXCLUDE_DIRS,
|
|
320
417
|
symbol,
|
|
321
418
|
projectDir,
|
|
322
|
-
]
|
|
323
|
-
if (
|
|
324
|
-
if (result.error)
|
|
325
|
-
this.logger.error('SearcherService: grep failed', { method: 'grepSymbol', reason: result.error.message });
|
|
419
|
+
]);
|
|
420
|
+
if (!result) {
|
|
326
421
|
return { matches: [], truncated: false };
|
|
327
422
|
}
|
|
328
423
|
const excludeAbsPath = path.join(this.config.workspaceRoot, excludeFile);
|
|
329
|
-
const all = result.
|
|
424
|
+
const all = result.split('\n').flatMap(line => {
|
|
330
425
|
const match = line.match(/^(.+?):(\d+):/);
|
|
331
426
|
if (!match)
|
|
332
427
|
return [];
|
|
@@ -345,7 +440,7 @@ class SearcherService {
|
|
|
345
440
|
if (!/^[\w$]+$/.test(name))
|
|
346
441
|
throw new Error('Invalid symbol name');
|
|
347
442
|
if (kind === 'method')
|
|
348
|
-
return
|
|
443
|
+
return this.grepFindSymbol(name, project, kind);
|
|
349
444
|
return this.findSymbolFromIndex(name, project, kind);
|
|
350
445
|
}
|
|
351
446
|
async findSymbolFromIndex(name, project, kind) {
|
|
@@ -385,29 +480,25 @@ class SearcherService {
|
|
|
385
480
|
return this.grepFindSymbol(name, project, kind);
|
|
386
481
|
}
|
|
387
482
|
}
|
|
388
|
-
grepFindSymbol(name, project, kind) {
|
|
483
|
+
async grepFindSymbol(name, project, kind) {
|
|
389
484
|
const t0 = Date.now();
|
|
390
485
|
this.logger.debug('SearcherService: findSymbol', { name, project, kind, source: 'grep' });
|
|
391
486
|
const searchDir = project
|
|
392
|
-
?
|
|
487
|
+
? this.resolveProjectRoot(project)
|
|
393
488
|
: this.config.workspaceRoot;
|
|
394
489
|
const pattern = this.buildFindSymbolPattern(name, kind);
|
|
395
|
-
const result =
|
|
490
|
+
const result = await this.spawnGrep([
|
|
396
491
|
'-rn', '-E',
|
|
397
492
|
'-m', String(FIND_SYMBOL_RAW_LIMIT),
|
|
398
493
|
'--include=*.ts', '--include=*.tsx', '--include=*.js', '--include=*.jsx',
|
|
399
494
|
...GREP_EXCLUDE_DIRS,
|
|
400
495
|
pattern,
|
|
401
496
|
searchDir,
|
|
402
|
-
]
|
|
403
|
-
if (result
|
|
404
|
-
this.logger.error('SearcherService: grep failed', { method: 'findSymbol', reason: result.error.message });
|
|
405
|
-
return { hits: [], truncated: false };
|
|
406
|
-
}
|
|
407
|
-
if (!result.stdout)
|
|
497
|
+
]);
|
|
498
|
+
if (!result)
|
|
408
499
|
return { hits: [], truncated: false };
|
|
409
500
|
const hits = [];
|
|
410
|
-
for (const raw of result.
|
|
501
|
+
for (const raw of result.split('\n')) {
|
|
411
502
|
if (hits.length >= FIND_SYMBOL_LIMIT)
|
|
412
503
|
break;
|
|
413
504
|
const m = raw.match(/^(.+?):(\d+):(.*)$/);
|
|
@@ -434,7 +525,7 @@ class SearcherService {
|
|
|
434
525
|
listFiles(relativePath, project, depth, pattern) {
|
|
435
526
|
const t0 = Date.now();
|
|
436
527
|
this.logger.debug('SearcherService: listFiles', { relativePath, project, depth, pattern });
|
|
437
|
-
const projectRoot =
|
|
528
|
+
const projectRoot = this.resolveProjectRoot(project);
|
|
438
529
|
const resolvedTarget = path.resolve(projectRoot, relativePath);
|
|
439
530
|
const resolvedProject = path.resolve(projectRoot);
|
|
440
531
|
if (resolvedTarget !== resolvedProject && !resolvedTarget.startsWith(resolvedProject + path.sep)) {
|
|
@@ -512,25 +603,29 @@ class SearcherService {
|
|
|
512
603
|
let lineNo = 0;
|
|
513
604
|
let lastRead = 0;
|
|
514
605
|
let hasMore = false;
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
606
|
+
try {
|
|
607
|
+
for await (const line of rl) {
|
|
608
|
+
lineNo++;
|
|
609
|
+
if (lineNo >= startLine && lineNo <= endLine)
|
|
610
|
+
lines.push(line);
|
|
611
|
+
if (lineNo <= endLine)
|
|
612
|
+
lastRead = lineNo;
|
|
613
|
+
if (lineNo > endLine) {
|
|
614
|
+
hasMore = true;
|
|
615
|
+
rl.close();
|
|
616
|
+
stream.destroy();
|
|
617
|
+
break;
|
|
618
|
+
}
|
|
526
619
|
}
|
|
527
620
|
}
|
|
621
|
+
catch (err) {
|
|
622
|
+
rl.close();
|
|
623
|
+
stream.destroy();
|
|
624
|
+
throw err;
|
|
625
|
+
}
|
|
528
626
|
this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
|
|
529
627
|
return { ok: true, filePath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
|
|
530
628
|
}
|
|
531
|
-
isNonEnglish(text) {
|
|
532
|
-
return /[^\x00-\x7F]/.test(text);
|
|
533
|
-
}
|
|
534
629
|
walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
|
|
535
630
|
let entries;
|
|
536
631
|
try {
|
|
@@ -590,5 +685,40 @@ class SearcherService {
|
|
|
590
685
|
return 'method';
|
|
591
686
|
return 'variable';
|
|
592
687
|
}
|
|
688
|
+
/**
|
|
689
|
+
* Async wrapper around grep to avoid blocking the event loop.
|
|
690
|
+
* Returns stdout as a string, or null if grep returned no results or errored.
|
|
691
|
+
*/
|
|
692
|
+
spawnGrep(args) {
|
|
693
|
+
return new Promise((resolve) => {
|
|
694
|
+
const proc = (0, child_process_1.spawn)('grep', args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
695
|
+
const chunks = [];
|
|
696
|
+
let totalBytes = 0;
|
|
697
|
+
const MAX_BUFFER = 10 * 1024 * 1024;
|
|
698
|
+
proc.stdout.on('data', (chunk) => {
|
|
699
|
+
totalBytes += chunk.length;
|
|
700
|
+
if (totalBytes <= MAX_BUFFER) {
|
|
701
|
+
chunks.push(chunk);
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
proc.stderr.on('data', () => { });
|
|
705
|
+
proc.on('error', (err) => {
|
|
706
|
+
this.logger.error('SearcherService: grep spawn failed', { reason: err.message });
|
|
707
|
+
resolve(null);
|
|
708
|
+
});
|
|
709
|
+
proc.on('close', (code) => {
|
|
710
|
+
// grep returns 1 for "no matches" — not an error
|
|
711
|
+
if (code !== 0 && code !== 1) {
|
|
712
|
+
this.logger.warn('SearcherService: grep exited with code', { code });
|
|
713
|
+
}
|
|
714
|
+
if (chunks.length === 0) {
|
|
715
|
+
resolve(null);
|
|
716
|
+
}
|
|
717
|
+
else {
|
|
718
|
+
resolve(Buffer.concat(chunks).toString('utf-8'));
|
|
719
|
+
}
|
|
720
|
+
});
|
|
721
|
+
});
|
|
722
|
+
}
|
|
593
723
|
}
|
|
594
724
|
exports.SearcherService = SearcherService;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
5
|
+
/**
|
|
6
|
+
* Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
|
|
7
|
+
* of strings, `input_type` (`query` | `document`) is required for best quality,
|
|
8
|
+
* and the optional `output_dimension` requests Matryoshka truncation. Response
|
|
9
|
+
* is `{ data: [{ embedding, index }] }`.
|
|
10
|
+
*/
|
|
11
|
+
export declare class VoyageEmbedder implements IEmbedder {
|
|
12
|
+
private config;
|
|
13
|
+
private logger;
|
|
14
|
+
readonly dimensions: number;
|
|
15
|
+
rerank?: (query: string, documents: string[], topN: number) => Promise<number[]>;
|
|
16
|
+
private readonly url;
|
|
17
|
+
private readonly doFetch;
|
|
18
|
+
private readonly baseDelayMs?;
|
|
19
|
+
constructor(config: Config, logger?: ILogger, deps?: EmbedderDeps);
|
|
20
|
+
embed(text: string, inputType?: InputType): Promise<number[]>;
|
|
21
|
+
embedBatch(texts: string[], inputType?: InputType): Promise<number[][]>;
|
|
22
|
+
}
|