@esankhan3/code-search-mcp 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +203 -263
- package/dist/resources/resources.js +3 -3
- package/dist/server.js +2 -2
- package/dist/tools/graph.js +1 -1
- package/dist/tools/index-tools.js +1 -1
- package/dist/tools/profile.js +2 -2
- package/dist/tools/search.js +1 -1
- package/package.json +16 -9
- package/dist/core/ast-graph-builder.js +0 -800
- package/dist/core/chunker.js +0 -368
- package/dist/core/claude-runner.js +0 -173
- package/dist/core/config.js +0 -59
- package/dist/core/cross-repo-detector.js +0 -887
- package/dist/core/embedder.js +0 -334
- package/dist/core/file-walker.js +0 -186
- package/dist/core/git-diff.js +0 -160
- package/dist/core/graph-metrics.js +0 -233
- package/dist/core/indexer.js +0 -552
- package/dist/core/project-graph-builder.js +0 -202
- package/dist/core/query-classifier.js +0 -144
- package/dist/core/query-router.js +0 -210
- package/dist/core/rag-evaluator.js +0 -213
- package/dist/core/repo-profiler.js +0 -383
- package/dist/core/reranker.js +0 -218
- package/dist/core/retriever.js +0 -242
- package/dist/core/semantic-edge-detector.js +0 -147
- package/dist/core/service-mesh-inferrer.js +0 -241
- package/dist/core/structural-hasher.js +0 -316
- package/dist/core/tree-sitter-parser.js +0 -539
- package/dist/core/types.js +0 -0
- package/dist/core/vector-store.js +0 -259
- package/dist/core/workspace-detector.js +0 -481
package/dist/core/indexer.js
DELETED
|
@@ -1,552 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync, mkdirSync, writeFileSync, readdirSync, statSync } from "node:fs";
|
|
2
|
-
import { execSync } from "node:child_process";
|
|
3
|
-
import { join, basename } from "node:path";
|
|
4
|
-
import { chunkRepo, chunkChangedFiles } from "./chunker.js";
|
|
5
|
-
import { buildAstGraph, generateGraphReport, incrementalGraphUpdate } from "./ast-graph-builder.js";
|
|
6
|
-
import { getAllChanges, getChangedFilesList, getDeletedFilesList } from "./git-diff.js";
|
|
7
|
-
import { createEmbeddingProvider } from "./embedder.js";
|
|
8
|
-
import { VectorStore } from "./vector-store.js";
|
|
9
|
-
import { ProjectGraphBuilder } from "./project-graph-builder.js";
|
|
10
|
-
import { detectCrossRepoEdges } from "./cross-repo-detector.js";
|
|
11
|
-
import { detectWorkspace } from "./workspace-detector.js";
|
|
12
|
-
import { HybridRetriever } from "./retriever.js";
|
|
13
|
-
import { loadKnowledgeConfig, getKnowledgeBasePath, DEFAULT_CONFIG } from "./config.js";
|
|
14
|
-
import { profileProject, loadAllProfiles } from "./repo-profiler.js";
|
|
15
|
-
import { inferServiceMesh } from "./service-mesh-inferrer.js";
|
|
16
|
-
import { deduplicateByStructure } from "./structural-hasher.js";
|
|
17
|
-
import { createQueryRouter } from "./query-router.js";
|
|
18
|
-
function getRepoSha(repoPath) {
|
|
19
|
-
try {
|
|
20
|
-
return execSync("git rev-parse HEAD", { cwd: repoPath, stdio: "pipe", encoding: "utf-8", timeout: 5e3 }).trim();
|
|
21
|
-
} catch {
|
|
22
|
-
return null;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
function readRepoIndexMeta(basePath, repoName) {
|
|
26
|
-
const metaPath = join(basePath, repoName, "index_meta.json");
|
|
27
|
-
if (!existsSync(metaPath)) return null;
|
|
28
|
-
try {
|
|
29
|
-
return JSON.parse(readFileSync(metaPath, "utf-8"));
|
|
30
|
-
} catch {
|
|
31
|
-
return null;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
function writeRepoIndexMeta(basePath, repoName, meta) {
|
|
35
|
-
const dir = join(basePath, repoName);
|
|
36
|
-
mkdirSync(dir, { recursive: true });
|
|
37
|
-
writeFileSync(join(dir, "index_meta.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
38
|
-
}
|
|
39
|
-
class KnowledgeIndexer {
|
|
40
|
-
// ---------------------------------------------------------------------------
|
|
41
|
-
// BUILD KB — fast, static, no embedding. Profiles + chunks + graphs + edges.
|
|
42
|
-
// ---------------------------------------------------------------------------
|
|
43
|
-
async buildKB(project, repos, config, opts) {
|
|
44
|
-
const log = opts?.onProgress ?? (() => {
|
|
45
|
-
});
|
|
46
|
-
const report = opts?.onDetailedProgress ?? (() => {
|
|
47
|
-
});
|
|
48
|
-
const startTime = Date.now();
|
|
49
|
-
const basePath = getKnowledgeBasePath(project);
|
|
50
|
-
mkdirSync(basePath, { recursive: true });
|
|
51
|
-
const reposToIndex = [];
|
|
52
|
-
const skippedRepos = [];
|
|
53
|
-
for (const repo of repos) {
|
|
54
|
-
if (opts?.force) {
|
|
55
|
-
reposToIndex.push(repo);
|
|
56
|
-
continue;
|
|
57
|
-
}
|
|
58
|
-
const currentSha = getRepoSha(repo.path);
|
|
59
|
-
const meta = readRepoIndexMeta(basePath, repo.name);
|
|
60
|
-
if (meta && currentSha && meta.lastIndexedSha === currentSha) {
|
|
61
|
-
skippedRepos.push(repo.name);
|
|
62
|
-
log(`Skipping ${repo.name} \u2014 unchanged (${currentSha.slice(0, 7)})`);
|
|
63
|
-
} else {
|
|
64
|
-
reposToIndex.push(repo);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
if (reposToIndex.length === 0) {
|
|
68
|
-
log("All repos up to date \u2014 nothing to build.");
|
|
69
|
-
report({ phase: "done", message: "All repos up to date", percent: 100, etaSeconds: 0, skippedRepos });
|
|
70
|
-
return { project, repos: [], totalChunks: 0, totalTokens: 0, crossRepoEdges: 0, durationMs: 0, chunksPath: join(basePath, "chunks.json") };
|
|
71
|
-
}
|
|
72
|
-
const { isLlmAvailable } = await import("./claude-runner.js");
|
|
73
|
-
if (isLlmAvailable()) {
|
|
74
|
-
report({ phase: "profiling", message: `Profiling ${repos.length} repos with LLM...`, percent: 3, etaSeconds: -1, reposTotal: repos.length, reposProcessed: 0 });
|
|
75
|
-
log(`Profiling ${repos.length} repos with LLM...`);
|
|
76
|
-
try {
|
|
77
|
-
const profiles = await profileProject(project, repos, {
|
|
78
|
-
force: opts?.force,
|
|
79
|
-
onProgress: (m) => {
|
|
80
|
-
log(m);
|
|
81
|
-
report({ phase: "profiling", message: m, percent: 5, etaSeconds: -1 });
|
|
82
|
-
}
|
|
83
|
-
});
|
|
84
|
-
log(`Profiled ${profiles.length} repos`);
|
|
85
|
-
} catch (err) {
|
|
86
|
-
const errMsg = err instanceof Error ? err.message : String(err);
|
|
87
|
-
log(`Warning: Repo profiling skipped: ${errMsg}`);
|
|
88
|
-
report({ phase: "profiling", message: `Profiling skipped: ${errMsg.slice(0, 100)}`, percent: 8, etaSeconds: -1 });
|
|
89
|
-
}
|
|
90
|
-
} else {
|
|
91
|
-
log("LLM profiling disabled (CODE_SEARCH_LLM_MODE=none) \u2014 skipping");
|
|
92
|
-
report({ phase: "profiling", message: "Skipped (LLM disabled)", percent: 8, etaSeconds: -1 });
|
|
93
|
-
}
|
|
94
|
-
report({ phase: "chunking", message: `Chunking ${reposToIndex.length} repos...`, percent: 10, etaSeconds: -1, reposTotal: repos.length, reposProcessed: skippedRepos.length, skippedRepos });
|
|
95
|
-
log(`Chunking ${reposToIndex.length} repos (${skippedRepos.length} skipped \u2014 unchanged)...`);
|
|
96
|
-
const allChunks = [];
|
|
97
|
-
const repoStats = [];
|
|
98
|
-
const repoChunkResults = /* @__PURE__ */ new Map();
|
|
99
|
-
const repoDiffs = /* @__PURE__ */ new Map();
|
|
100
|
-
for (const repo of reposToIndex) {
|
|
101
|
-
const meta = opts?.force ? null : readRepoIndexMeta(basePath, repo.name);
|
|
102
|
-
const diff = opts?.force ? null : meta?.lastIndexedSha ? getAllChanges(repo.path, meta.lastIndexedSha) : null;
|
|
103
|
-
const useIncremental = diff && !diff.fallbackToFull && diff.added.length + diff.modified.length + diff.deleted.length > 0;
|
|
104
|
-
let result;
|
|
105
|
-
if (useIncremental) {
|
|
106
|
-
const changedCount = diff.added.length + diff.modified.length;
|
|
107
|
-
const deletedCount = diff.deleted.length;
|
|
108
|
-
log(` ${repo.name}: git diff \u2192 ${changedCount} changed, ${deletedCount} deleted (incremental)`);
|
|
109
|
-
result = await chunkChangedFiles(repo.path, repo.name, project, config.chunking, diff);
|
|
110
|
-
repoDiffs.set(repo.name, diff);
|
|
111
|
-
} else {
|
|
112
|
-
result = await chunkRepo(repo.path, repo.name, project, config.chunking, meta?.files ?? void 0);
|
|
113
|
-
}
|
|
114
|
-
allChunks.push(...result.chunks);
|
|
115
|
-
repoChunkResults.set(repo.name, result);
|
|
116
|
-
const totalChunkCount = Object.values(result.fileIndex).reduce((sum, f) => sum + f.chunkCount, 0);
|
|
117
|
-
repoStats.push({ name: repo.name, chunkCount: totalChunkCount, language: repo.language });
|
|
118
|
-
}
|
|
119
|
-
for (const name of skippedRepos) {
|
|
120
|
-
const meta = readRepoIndexMeta(basePath, name);
|
|
121
|
-
repoStats.push({ name, chunkCount: meta?.chunkCount ?? 0, language: "" });
|
|
122
|
-
}
|
|
123
|
-
log(`Chunked ${allChunks.length} chunks from ${reposToIndex.length} repos`);
|
|
124
|
-
report({ phase: "dedup", message: "Deduplicating chunks by structure...", percent: 25, etaSeconds: -1 });
|
|
125
|
-
const dedupResult = deduplicateByStructure(allChunks);
|
|
126
|
-
const uniqueChunks = dedupResult.unique;
|
|
127
|
-
if (dedupResult.savings.chunks > 0) {
|
|
128
|
-
log(`Structural dedup: ${dedupResult.savings.chunks} duplicates removed`);
|
|
129
|
-
}
|
|
130
|
-
const workspaceMaps = /* @__PURE__ */ new Map();
|
|
131
|
-
for (const repo of repos) {
|
|
132
|
-
try {
|
|
133
|
-
const wsMap = detectWorkspace(repo.path);
|
|
134
|
-
if (wsMap.packages.length > 0) {
|
|
135
|
-
workspaceMaps.set(repo.name, wsMap);
|
|
136
|
-
log(`Detected workspace in ${repo.name}: ${wsMap.packages.length} packages`);
|
|
137
|
-
}
|
|
138
|
-
} catch (err) {
|
|
139
|
-
log(`Warning: Workspace detection failed for ${repo.name}: ${err}`);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
report({ phase: "graphing", message: "Building AST graphs...", percent: 40, etaSeconds: -1 });
|
|
143
|
-
const graphBuilder = new ProjectGraphBuilder();
|
|
144
|
-
await graphBuilder.init();
|
|
145
|
-
for (const repo of repos) {
|
|
146
|
-
try {
|
|
147
|
-
const repoKbDir = join(basePath, repo.name);
|
|
148
|
-
mkdirSync(repoKbDir, { recursive: true });
|
|
149
|
-
const existingGraphPath = join(repoKbDir, "graph.json");
|
|
150
|
-
const diff = repoDiffs.get(repo.name);
|
|
151
|
-
let graph;
|
|
152
|
-
if (diff && !diff.fallbackToFull && existsSync(existingGraphPath)) {
|
|
153
|
-
const existingGraph = JSON.parse(readFileSync(existingGraphPath, "utf-8"));
|
|
154
|
-
graph = await incrementalGraphUpdate(
|
|
155
|
-
existingGraph,
|
|
156
|
-
getChangedFilesList(diff),
|
|
157
|
-
getDeletedFilesList(diff),
|
|
158
|
-
repo.path,
|
|
159
|
-
{ workspaceMap: workspaceMaps.get(repo.name) }
|
|
160
|
-
);
|
|
161
|
-
log(`Updated AST graph for ${repo.name} incrementally (${diff.added.length + diff.modified.length} files changed)`);
|
|
162
|
-
} else {
|
|
163
|
-
graph = await buildAstGraph(repo.path, { workspaceMap: workspaceMaps.get(repo.name) });
|
|
164
|
-
log(`Built AST graph for ${repo.name} (${graph.nodes.length} nodes, ${graph.links.length} edges)`);
|
|
165
|
-
}
|
|
166
|
-
graphBuilder.addRepoGraph(repo.name, graph);
|
|
167
|
-
writeFileSync(join(repoKbDir, "graph.json"), JSON.stringify(graph));
|
|
168
|
-
writeFileSync(join(repoKbDir, "GRAPH_REPORT.md"), generateGraphReport(repo.name, graph));
|
|
169
|
-
} catch (err) {
|
|
170
|
-
log(`Warning: AST graph build failed for ${repo.name}: ${err}`);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
report({ phase: "graphing", message: "Detecting cross-repo edges...", percent: 70, etaSeconds: -1 });
|
|
174
|
-
let crossRepoEdgeCount = 0;
|
|
175
|
-
const hasWorkspaces = workspaceMaps.size > 0;
|
|
176
|
-
if (repos.length > 1 || hasWorkspaces) {
|
|
177
|
-
const crossEdges = await detectCrossRepoEdges(repos, workspaceMaps);
|
|
178
|
-
graphBuilder.addCrossRepoEdges(crossEdges);
|
|
179
|
-
crossRepoEdgeCount = crossEdges.length;
|
|
180
|
-
log(`Detected ${crossEdges.length} cross-repo edges`);
|
|
181
|
-
}
|
|
182
|
-
if (isLlmAvailable()) {
|
|
183
|
-
report({ phase: "service-mesh", message: "Inferring service mesh from profiles...", percent: 80, etaSeconds: -1 });
|
|
184
|
-
try {
|
|
185
|
-
const profiles = loadAllProfiles(project);
|
|
186
|
-
if (profiles.length > 0) {
|
|
187
|
-
log(`Inferring service mesh from ${profiles.length} profiles...`);
|
|
188
|
-
const meshEdges = await inferServiceMesh(profiles, {
|
|
189
|
-
onProgress: (m) => {
|
|
190
|
-
log(m);
|
|
191
|
-
report({ phase: "service-mesh", message: m, percent: 82, etaSeconds: -1 });
|
|
192
|
-
}
|
|
193
|
-
});
|
|
194
|
-
if (meshEdges.length > 0) {
|
|
195
|
-
graphBuilder.addCrossRepoEdges(meshEdges);
|
|
196
|
-
crossRepoEdgeCount += meshEdges.length;
|
|
197
|
-
log(`Service mesh: ${meshEdges.length} edges inferred`);
|
|
198
|
-
}
|
|
199
|
-
} else {
|
|
200
|
-
log("No profiles found \u2014 skipping service mesh inference");
|
|
201
|
-
}
|
|
202
|
-
} catch (err) {
|
|
203
|
-
log(`Warning: Service mesh inference failed (non-fatal): ${err}`);
|
|
204
|
-
}
|
|
205
|
-
} else {
|
|
206
|
-
log("LLM service mesh inference disabled \u2014 skipping");
|
|
207
|
-
report({ phase: "service-mesh", message: "Skipped (LLM disabled)", percent: 85, etaSeconds: -1 });
|
|
208
|
-
}
|
|
209
|
-
report({ phase: "graphing", message: "Detecting communities...", percent: 90, etaSeconds: -1 });
|
|
210
|
-
graphBuilder.detectCommunities();
|
|
211
|
-
const graphOutputPath = join(basePath, "system_graph_v2.json");
|
|
212
|
-
writeFileSync(graphOutputPath, JSON.stringify(graphBuilder.exportJson(), null, 2));
|
|
213
|
-
log(`Saved project graph to ${graphOutputPath}`);
|
|
214
|
-
const chunksPath = join(basePath, "chunks.json");
|
|
215
|
-
writeFileSync(chunksPath, JSON.stringify(uniqueChunks));
|
|
216
|
-
log(`Saved ${uniqueChunks.length} chunks to ${chunksPath}`);
|
|
217
|
-
const allDeletedFiles = [];
|
|
218
|
-
for (const repo of reposToIndex) {
|
|
219
|
-
const result = repoChunkResults.get(repo.name);
|
|
220
|
-
if (result?.deletedFiles) {
|
|
221
|
-
for (const f of result.deletedFiles) {
|
|
222
|
-
allDeletedFiles.push({ repoName: repo.name, filePath: f });
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
const deletedPath = join(basePath, "deleted_files.json");
|
|
227
|
-
writeFileSync(deletedPath, JSON.stringify(allDeletedFiles));
|
|
228
|
-
for (const repo of reposToIndex) {
|
|
229
|
-
const sha = getRepoSha(repo.path);
|
|
230
|
-
const result = repoChunkResults.get(repo.name);
|
|
231
|
-
const totalChunkCount = result ? Object.values(result.fileIndex).reduce((sum, f) => sum + f.chunkCount, 0) : 0;
|
|
232
|
-
if (sha) {
|
|
233
|
-
writeRepoIndexMeta(basePath, repo.name, {
|
|
234
|
-
lastIndexedSha: sha,
|
|
235
|
-
lastIndexedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
236
|
-
chunkCount: totalChunkCount,
|
|
237
|
-
embeddingProvider: "pending",
|
|
238
|
-
files: result?.fileIndex
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
const durationMs = Date.now() - startTime;
|
|
243
|
-
report({ phase: "done", message: `KB built: ${uniqueChunks.length} chunks, ${crossRepoEdgeCount} edges in ${formatEta(Math.ceil(durationMs / 1e3))}`, percent: 100, etaSeconds: 0, skippedRepos });
|
|
244
|
-
return {
|
|
245
|
-
project,
|
|
246
|
-
repos: repoStats,
|
|
247
|
-
totalChunks: uniqueChunks.length,
|
|
248
|
-
totalTokens: uniqueChunks.reduce((sum, c) => sum + c.tokens, 0),
|
|
249
|
-
crossRepoEdges: crossRepoEdgeCount,
|
|
250
|
-
durationMs,
|
|
251
|
-
chunksPath
|
|
252
|
-
};
|
|
253
|
-
}
|
|
254
|
-
// ---------------------------------------------------------------------------
|
|
255
|
-
// EMBED — incremental: only embeds new/changed chunks, preserves existing.
|
|
256
|
-
// ---------------------------------------------------------------------------
|
|
257
|
-
async embedChunks(project, config, opts) {
|
|
258
|
-
const log = opts?.onProgress ?? (() => {
|
|
259
|
-
});
|
|
260
|
-
const report = opts?.onDetailedProgress ?? (() => {
|
|
261
|
-
});
|
|
262
|
-
const startTime = Date.now();
|
|
263
|
-
const basePath = getKnowledgeBasePath(project);
|
|
264
|
-
const chunksPath = join(basePath, "chunks.json");
|
|
265
|
-
if (!existsSync(chunksPath)) {
|
|
266
|
-
throw new Error(`No chunks found \u2014 run Build KB first. Expected: ${chunksPath}`);
|
|
267
|
-
}
|
|
268
|
-
const chunks = JSON.parse(readFileSync(chunksPath, "utf-8"));
|
|
269
|
-
log(`Loaded ${chunks.length} chunks from ${chunksPath}`);
|
|
270
|
-
const dbPath = join(basePath, "lancedb");
|
|
271
|
-
const vectorStore = new VectorStore(dbPath);
|
|
272
|
-
await vectorStore.init();
|
|
273
|
-
const existingIds = /* @__PURE__ */ new Set();
|
|
274
|
-
try {
|
|
275
|
-
const stats = await vectorStore.getStats();
|
|
276
|
-
if (stats && stats.rowCount > 0) {
|
|
277
|
-
const existingChunks = await vectorStore.getChunkIds(project);
|
|
278
|
-
for (const id of existingChunks) existingIds.add(id);
|
|
279
|
-
}
|
|
280
|
-
} catch {
|
|
281
|
-
}
|
|
282
|
-
const newChunks = chunks.filter((c) => !existingIds.has(c.id));
|
|
283
|
-
const deletedFiles = this.getDeletedFiles(basePath);
|
|
284
|
-
if (newChunks.length === 0 && deletedFiles.length === 0) {
|
|
285
|
-
log("All chunks already embedded \u2014 nothing to do.");
|
|
286
|
-
report({ phase: "done", message: "All chunks already embedded", percent: 100, etaSeconds: 0 });
|
|
287
|
-
const repoNames2 = [...new Set(chunks.map((c) => c.repoName))];
|
|
288
|
-
return {
|
|
289
|
-
project,
|
|
290
|
-
repos: repoNames2.map((n) => ({ name: n, chunkCount: chunks.filter((c) => c.repoName === n).length, language: "" })),
|
|
291
|
-
totalChunks: chunks.length,
|
|
292
|
-
totalTokens: chunks.reduce((sum, c) => sum + c.tokens, 0),
|
|
293
|
-
embeddingProvider: "cached",
|
|
294
|
-
embeddingDimensions: 0,
|
|
295
|
-
crossRepoEdges: 0,
|
|
296
|
-
lastIndexed: (/* @__PURE__ */ new Date()).toISOString(),
|
|
297
|
-
indexDurationMs: Date.now() - startTime
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
|
-
if (deletedFiles.length > 0) {
|
|
301
|
-
const byRepo = /* @__PURE__ */ new Map();
|
|
302
|
-
for (const d of deletedFiles) {
|
|
303
|
-
const list = byRepo.get(d.repoName) ?? [];
|
|
304
|
-
list.push(d.filePath);
|
|
305
|
-
byRepo.set(d.repoName, list);
|
|
306
|
-
}
|
|
307
|
-
for (const [repoName, filePaths] of byRepo) {
|
|
308
|
-
log(`Removing chunks for ${filePaths.length} deleted files from ${repoName}...`);
|
|
309
|
-
await vectorStore.deleteFileChunks(project, repoName, filePaths);
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
const embedder = createEmbeddingProvider(config.embedding);
|
|
313
|
-
const isOllama = embedder.name === "ollama";
|
|
314
|
-
const batchSize = isOllama ? 10 : 50;
|
|
315
|
-
const batchDelay = isOllama ? 50 : 100;
|
|
316
|
-
log(`Embedding ${newChunks.length} new chunks with ${embedder.name} (${chunks.length - newChunks.length} cached, batch size: ${batchSize})...`);
|
|
317
|
-
const texts = newChunks.map((c) => c.contextualizedContent);
|
|
318
|
-
const embeddings = [];
|
|
319
|
-
const totalBatches = Math.ceil(texts.length / batchSize);
|
|
320
|
-
let batchesDone = 0;
|
|
321
|
-
const embedStartTime = Date.now();
|
|
322
|
-
for (let i = 0; i < texts.length; i += batchSize) {
|
|
323
|
-
const batch = texts.slice(i, i + batchSize);
|
|
324
|
-
const batchEmbeddings = await embedder.embed(batch);
|
|
325
|
-
embeddings.push(...batchEmbeddings);
|
|
326
|
-
batchesDone++;
|
|
327
|
-
const elapsed = Date.now() - embedStartTime;
|
|
328
|
-
const msPerBatch = elapsed / batchesDone;
|
|
329
|
-
const remainingBatches = totalBatches - batchesDone;
|
|
330
|
-
const etaSeconds = Math.ceil(msPerBatch * remainingBatches / 1e3);
|
|
331
|
-
const percent = Math.round(5 + batchesDone / totalBatches * 85);
|
|
332
|
-
const processed = Math.min(i + batchSize, texts.length);
|
|
333
|
-
report({
|
|
334
|
-
phase: "embedding",
|
|
335
|
-
message: `Embedding: ${processed}/${texts.length} new (~${etaSeconds}s remaining)`,
|
|
336
|
-
percent,
|
|
337
|
-
etaSeconds,
|
|
338
|
-
chunksTotal: texts.length,
|
|
339
|
-
chunksProcessed: processed
|
|
340
|
-
});
|
|
341
|
-
if (batchesDone % 10 === 0 || batchesDone === totalBatches) {
|
|
342
|
-
log(` Embedded ${processed}/${texts.length} (ETA: ${formatEta(etaSeconds)})`);
|
|
343
|
-
}
|
|
344
|
-
if (i + batchSize < texts.length && batchDelay > 0) {
|
|
345
|
-
await new Promise((resolve) => setTimeout(resolve, batchDelay));
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
const embeddedChunks = newChunks.map((chunk, i) => ({ ...chunk, embedding: embeddings[i] }));
|
|
349
|
-
report({ phase: "storing", message: "Saving new chunks to vector database...", percent: 92, etaSeconds: -1 });
|
|
350
|
-
if (embeddedChunks.length > 0) {
|
|
351
|
-
await vectorStore.addChunks(embeddedChunks);
|
|
352
|
-
}
|
|
353
|
-
log(`Stored ${embeddedChunks.length} new chunks in LanceDB (${deletedIds.length} removed)`);
|
|
354
|
-
const repoNames = [...new Set(chunks.map((c) => c.repoName))];
|
|
355
|
-
for (const repoName of repoNames) {
|
|
356
|
-
const metaPath = join(basePath, repoName, "index_meta.json");
|
|
357
|
-
if (existsSync(metaPath)) {
|
|
358
|
-
try {
|
|
359
|
-
const meta = JSON.parse(readFileSync(metaPath, "utf-8"));
|
|
360
|
-
meta.embeddingProvider = embedder.name;
|
|
361
|
-
meta.lastIndexedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
362
|
-
writeFileSync(metaPath, JSON.stringify(meta, null, 2), "utf-8");
|
|
363
|
-
} catch {
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
const durationMs = Date.now() - startTime;
|
|
368
|
-
report({ phase: "done", message: `Embedded ${newChunks.length} new chunks (${deletedIds.length} removed) in ${formatEta(Math.ceil(durationMs / 1e3))}`, percent: 100, etaSeconds: 0 });
|
|
369
|
-
return {
|
|
370
|
-
project,
|
|
371
|
-
repos: repoNames.map((n) => ({ name: n, chunkCount: chunks.filter((c) => c.repoName === n).length, language: "" })),
|
|
372
|
-
totalChunks: chunks.length,
|
|
373
|
-
totalTokens: chunks.reduce((sum, c) => sum + c.tokens, 0),
|
|
374
|
-
embeddingProvider: embedder.name,
|
|
375
|
-
embeddingDimensions: embedder.dimensions,
|
|
376
|
-
crossRepoEdges: 0,
|
|
377
|
-
lastIndexed: (/* @__PURE__ */ new Date()).toISOString(),
|
|
378
|
-
indexDurationMs: durationMs
|
|
379
|
-
};
|
|
380
|
-
}
|
|
381
|
-
/** Read the deleted files list saved by buildKB */
|
|
382
|
-
getDeletedFiles(basePath) {
|
|
383
|
-
const deletedPath = join(basePath, "deleted_files.json");
|
|
384
|
-
if (!existsSync(deletedPath)) return [];
|
|
385
|
-
try {
|
|
386
|
-
return JSON.parse(readFileSync(deletedPath, "utf-8"));
|
|
387
|
-
} catch {
|
|
388
|
-
return [];
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
// ---------------------------------------------------------------------------
|
|
392
|
-
// FULL INDEX — convenience: buildKB + embedChunks in one call
|
|
393
|
-
// ---------------------------------------------------------------------------
|
|
394
|
-
async indexProject(project, repos, config, opts) {
|
|
395
|
-
await this.buildKB(project, repos, config, opts);
|
|
396
|
-
return this.embedChunks(project, config, opts);
|
|
397
|
-
}
|
|
398
|
-
/** Load index statistics for a project */
|
|
399
|
-
async getStats(project) {
|
|
400
|
-
const basePath = getKnowledgeBasePath(project);
|
|
401
|
-
const dbPath = join(basePath, "lancedb");
|
|
402
|
-
const store = new VectorStore(dbPath);
|
|
403
|
-
await store.init();
|
|
404
|
-
const stats = await store.getStats();
|
|
405
|
-
let provider = "unknown";
|
|
406
|
-
let lastIndexed = "";
|
|
407
|
-
const repos = [];
|
|
408
|
-
try {
|
|
409
|
-
const { readdirSync: readdirSync2 } = await import("node:fs");
|
|
410
|
-
for (const entry of readdirSync2(basePath)) {
|
|
411
|
-
const meta = readRepoIndexMeta(basePath, entry);
|
|
412
|
-
if (meta) {
|
|
413
|
-
repos.push({ name: entry, chunkCount: meta.chunkCount, language: "" });
|
|
414
|
-
if (meta.embeddingProvider !== "unknown") provider = meta.embeddingProvider;
|
|
415
|
-
if (meta.lastIndexedAt > lastIndexed) lastIndexed = meta.lastIndexedAt;
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
} catch {
|
|
419
|
-
}
|
|
420
|
-
return {
|
|
421
|
-
project,
|
|
422
|
-
repos,
|
|
423
|
-
totalChunks: stats?.rowCount ?? 0,
|
|
424
|
-
totalTokens: 0,
|
|
425
|
-
embeddingProvider: provider,
|
|
426
|
-
embeddingDimensions: 0,
|
|
427
|
-
crossRepoEdges: 0,
|
|
428
|
-
lastIndexed,
|
|
429
|
-
indexDurationMs: 0
|
|
430
|
-
};
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
const SKIP_DIRS = /* @__PURE__ */ new Set([".git", "node_modules", "dist", ".next", "build", "__pycache__", ".venv", "vendor", "target"]);
|
|
434
|
-
function discoverRepos(directoryPath) {
|
|
435
|
-
if (!existsSync(directoryPath)) return [];
|
|
436
|
-
const repos = [];
|
|
437
|
-
if (existsSync(join(directoryPath, ".git"))) {
|
|
438
|
-
repos.push({
|
|
439
|
-
name: basename(directoryPath),
|
|
440
|
-
path: directoryPath,
|
|
441
|
-
language: detectLanguage(directoryPath)
|
|
442
|
-
});
|
|
443
|
-
return repos;
|
|
444
|
-
}
|
|
445
|
-
try {
|
|
446
|
-
for (const entry of readdirSync(directoryPath)) {
|
|
447
|
-
if (entry.startsWith(".") || SKIP_DIRS.has(entry)) continue;
|
|
448
|
-
const fullPath = join(directoryPath, entry);
|
|
449
|
-
try {
|
|
450
|
-
if (!statSync(fullPath).isDirectory()) continue;
|
|
451
|
-
} catch {
|
|
452
|
-
continue;
|
|
453
|
-
}
|
|
454
|
-
if (!existsSync(join(fullPath, ".git"))) continue;
|
|
455
|
-
repos.push({
|
|
456
|
-
name: entry,
|
|
457
|
-
path: fullPath,
|
|
458
|
-
language: detectLanguage(fullPath)
|
|
459
|
-
});
|
|
460
|
-
}
|
|
461
|
-
} catch {
|
|
462
|
-
}
|
|
463
|
-
return repos;
|
|
464
|
-
}
|
|
465
|
-
function detectLanguage(repoPath) {
|
|
466
|
-
if (existsSync(join(repoPath, "go.mod"))) return "go";
|
|
467
|
-
if (existsSync(join(repoPath, "Cargo.toml"))) return "rust";
|
|
468
|
-
if (existsSync(join(repoPath, "pom.xml")) || existsSync(join(repoPath, "build.gradle"))) return "java";
|
|
469
|
-
if (existsSync(join(repoPath, "composer.json"))) return "php";
|
|
470
|
-
if (existsSync(join(repoPath, "pyproject.toml")) || existsSync(join(repoPath, "setup.py"))) return "python";
|
|
471
|
-
if (existsSync(join(repoPath, "package.json"))) {
|
|
472
|
-
if (existsSync(join(repoPath, "tsconfig.json"))) return "typescript";
|
|
473
|
-
return "javascript";
|
|
474
|
-
}
|
|
475
|
-
return "unknown";
|
|
476
|
-
}
|
|
477
|
-
async function buildKBFromPath(projectName, directoryPath, opts) {
|
|
478
|
-
const log = opts?.onProgress ?? (() => {
|
|
479
|
-
});
|
|
480
|
-
log(`Scanning ${directoryPath} for repos...`);
|
|
481
|
-
const repos = discoverRepos(directoryPath);
|
|
482
|
-
if (repos.length === 0) throw new Error(`No git repos found in ${directoryPath}`);
|
|
483
|
-
log(`Discovered ${repos.length} repos`);
|
|
484
|
-
const indexer = new KnowledgeIndexer();
|
|
485
|
-
return indexer.buildKB(projectName, repos, DEFAULT_CONFIG, opts);
|
|
486
|
-
}
|
|
487
|
-
async function embedFromPath(projectName, opts) {
|
|
488
|
-
const indexer = new KnowledgeIndexer();
|
|
489
|
-
return indexer.embedChunks(projectName, DEFAULT_CONFIG, opts);
|
|
490
|
-
}
|
|
491
|
-
async function indexFromPath(projectName, directoryPath, opts) {
|
|
492
|
-
const log = opts?.onProgress ?? (() => {
|
|
493
|
-
});
|
|
494
|
-
log(`Scanning ${directoryPath} for repos...`);
|
|
495
|
-
const repos = discoverRepos(directoryPath);
|
|
496
|
-
if (repos.length === 0) throw new Error(`No git repos found in ${directoryPath}`);
|
|
497
|
-
log(`Discovered ${repos.length} repos`);
|
|
498
|
-
const indexer = new KnowledgeIndexer();
|
|
499
|
-
return indexer.indexProject(projectName, repos, DEFAULT_CONFIG, opts);
|
|
500
|
-
}
|
|
501
|
-
function formatEta(seconds) {
|
|
502
|
-
if (seconds < 0) return "...";
|
|
503
|
-
if (seconds < 60) return `${seconds}s`;
|
|
504
|
-
const mins = Math.floor(seconds / 60);
|
|
505
|
-
const secs = seconds % 60;
|
|
506
|
-
return `${mins}m ${secs}s`;
|
|
507
|
-
}
|
|
508
|
-
async function getRetriever(project) {
|
|
509
|
-
const config = loadKnowledgeConfig(project);
|
|
510
|
-
const basePath = getKnowledgeBasePath(project);
|
|
511
|
-
const dbPath = join(basePath, "lancedb");
|
|
512
|
-
const vectorStore = new VectorStore(dbPath);
|
|
513
|
-
await vectorStore.init();
|
|
514
|
-
let graph = null;
|
|
515
|
-
const graphPath = join(basePath, "system_graph_v2.json");
|
|
516
|
-
if (existsSync(graphPath)) {
|
|
517
|
-
try {
|
|
518
|
-
const graphData = JSON.parse(readFileSync(graphPath, "utf-8"));
|
|
519
|
-
graph = new ProjectGraphBuilder();
|
|
520
|
-
await graph.importJson(graphData);
|
|
521
|
-
} catch {
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
const embedder = createEmbeddingProvider(config.embedding);
|
|
525
|
-
let reranker = null;
|
|
526
|
-
try {
|
|
527
|
-
const { createReranker } = await import("./reranker.js");
|
|
528
|
-
reranker = createReranker(config.retrieval.reranker);
|
|
529
|
-
} catch {
|
|
530
|
-
}
|
|
531
|
-
let queryRouter = null;
|
|
532
|
-
try {
|
|
533
|
-
queryRouter = await createQueryRouter(project, embedder);
|
|
534
|
-
if (queryRouter) {
|
|
535
|
-
console.log(`[knowledge] Query router ready (${queryRouter.repoCount} repo profiles)`);
|
|
536
|
-
}
|
|
537
|
-
} catch {
|
|
538
|
-
}
|
|
539
|
-
return new HybridRetriever(vectorStore, embedder, graph, {
|
|
540
|
-
maxChunks: config.retrieval.maxChunks,
|
|
541
|
-
maxTokens: config.retrieval.maxTokens,
|
|
542
|
-
hybridWeights: config.retrieval.hybridWeights
|
|
543
|
-
}, reranker, queryRouter);
|
|
544
|
-
}
|
|
545
|
-
export {
|
|
546
|
-
KnowledgeIndexer,
|
|
547
|
-
buildKBFromPath,
|
|
548
|
-
discoverRepos,
|
|
549
|
-
embedFromPath,
|
|
550
|
-
getRetriever,
|
|
551
|
-
indexFromPath
|
|
552
|
-
};
|