ai-mind-map 1.6.2 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/change-tracker/watcher.d.ts.map +1 -1
- package/dist/change-tracker/watcher.js +1 -0
- package/dist/change-tracker/watcher.js.map +1 -1
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1396 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-graph/graph.d.ts +5 -0
- package/dist/knowledge-graph/graph.d.ts.map +1 -1
- package/dist/knowledge-graph/graph.js +13 -1
- package/dist/knowledge-graph/graph.js.map +1 -1
- package/dist/knowledge-graph/indexer.d.ts +9 -0
- package/dist/knowledge-graph/indexer.d.ts.map +1 -1
- package/dist/knowledge-graph/indexer.js +339 -289
- package/dist/knowledge-graph/indexer.js.map +1 -1
- package/dist/knowledge-graph/semantic-search.d.ts +7 -3
- package/dist/knowledge-graph/semantic-search.d.ts.map +1 -1
- package/dist/knowledge-graph/semantic-search.js +68 -16
- package/dist/knowledge-graph/semantic-search.js.map +1 -1
- package/dist/memory/decision-log.d.ts.map +1 -1
- package/dist/memory/decision-log.js +11 -0
- package/dist/memory/decision-log.js.map +1 -1
- package/dist/memory/persistent-memory.d.ts.map +1 -1
- package/dist/memory/persistent-memory.js +7 -0
- package/dist/memory/persistent-memory.js.map +1 -1
- package/dist/memory/shared-sync.d.ts.map +1 -1
- package/dist/memory/shared-sync.js +6 -2
- package/dist/memory/shared-sync.js.map +1 -1
- package/package.json +1 -1
|
@@ -31,6 +31,28 @@ export class Indexer {
|
|
|
31
31
|
config;
|
|
32
32
|
ig;
|
|
33
33
|
changelog = null;
|
|
34
|
+
// Async lock for indexing operations
|
|
35
|
+
_indexLock = Promise.resolve();
|
|
36
|
+
_lockRelease = null;
|
|
37
|
+
async acquireIndexLock() {
|
|
38
|
+
// Wait for any existing lock to release
|
|
39
|
+
await this._indexLock;
|
|
40
|
+
let release;
|
|
41
|
+
this._indexLock = new Promise(resolve => { release = resolve; });
|
|
42
|
+
this._lockRelease = release;
|
|
43
|
+
return () => {
|
|
44
|
+
this._lockRelease = null;
|
|
45
|
+
release();
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
get isIndexing() {
|
|
49
|
+
return this._lockRelease !== null;
|
|
50
|
+
}
|
|
51
|
+
// Optional watcher reference to pause/resume during full reindex
|
|
52
|
+
watcher = null;
|
|
53
|
+
setWatcher(watcher) {
|
|
54
|
+
this.watcher = watcher;
|
|
55
|
+
}
|
|
34
56
|
/**
|
|
35
57
|
* Per-project-root ignore patterns.
|
|
36
58
|
* When multiple projects are indexed, each keeps its own .gitignore rules.
|
|
@@ -201,114 +223,130 @@ export class Indexer {
|
|
|
201
223
|
* @returns Indexing statistics
|
|
202
224
|
*/
|
|
203
225
|
async fullIndex(onProgress) {
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
message: 'Scanning project for source files...',
|
|
222
|
-
});
|
|
223
|
-
const files = await this.scanFiles();
|
|
224
|
-
stats.filesScanned = files.length;
|
|
225
|
-
if (files.length === 0) {
|
|
226
|
+
const release = await this.acquireIndexLock();
|
|
227
|
+
try {
|
|
228
|
+
// Pause watcher to prevent races during full reindex
|
|
229
|
+
this.watcher?.pause();
|
|
230
|
+
const startTime = Date.now();
|
|
231
|
+
const stats = {
|
|
232
|
+
filesScanned: 0,
|
|
233
|
+
filesParsed: 0,
|
|
234
|
+
filesSkipped: 0,
|
|
235
|
+
filesDeleted: 0,
|
|
236
|
+
nodesCreated: 0,
|
|
237
|
+
edgesCreated: 0,
|
|
238
|
+
parseErrors: 0,
|
|
239
|
+
durationMs: 0,
|
|
240
|
+
languages: {},
|
|
241
|
+
};
|
|
242
|
+
// Phase 1: Scanning
|
|
226
243
|
onProgress?.({
|
|
227
|
-
phase: '
|
|
244
|
+
phase: 'scanning',
|
|
228
245
|
current: 0,
|
|
229
246
|
total: 0,
|
|
230
|
-
message: '
|
|
247
|
+
message: 'Scanning project for source files...',
|
|
231
248
|
});
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
249
|
+
const files = await this.scanFiles();
|
|
250
|
+
stats.filesScanned = files.length;
|
|
251
|
+
if (files.length === 0) {
|
|
252
|
+
onProgress?.({
|
|
253
|
+
phase: 'complete',
|
|
254
|
+
current: 0,
|
|
255
|
+
total: 0,
|
|
256
|
+
message: 'No source files found to index.',
|
|
257
|
+
});
|
|
258
|
+
stats.durationMs = Date.now() - startTime;
|
|
259
|
+
return stats;
|
|
260
|
+
}
|
|
261
|
+
// Clear only nodes belonging to THIS project (preserve other projects)
|
|
262
|
+
this.graph.clearProject(this.config.projectRoot);
|
|
263
|
+
// Phase 2: Parsing
|
|
245
264
|
onProgress?.({
|
|
246
265
|
phase: 'parsing',
|
|
247
|
-
current,
|
|
248
|
-
total,
|
|
249
|
-
|
|
250
|
-
message: `Parsed ${current}/${total} files`,
|
|
266
|
+
current: 0,
|
|
267
|
+
total: files.length,
|
|
268
|
+
message: `Parsing ${files.length} files...`,
|
|
251
269
|
});
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
stats.
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
phase: 'storing',
|
|
284
|
-
current: i + 1,
|
|
285
|
-
total: parseResults.length,
|
|
286
|
-
currentFile: result.filePath,
|
|
287
|
-
message: `Stored ${i + 1}/${parseResults.length} files`,
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
|
-
// Memory pressure check every 100 files
|
|
291
|
-
if (i % 100 === 0 && i > 0) {
|
|
292
|
-
const mem = process.memoryUsage();
|
|
293
|
-
if (mem.heapUsed / mem.heapTotal > MEMORY_PRESSURE_THRESHOLD) {
|
|
294
|
-
process.stderr.write(`⚠ Memory pressure during fullIndex at file ${i}/${parseResults.length}. Stopping early.\n`);
|
|
295
|
-
break;
|
|
270
|
+
const parseResults = await parseFiles(files, 16, (current, total) => {
|
|
271
|
+
onProgress?.({
|
|
272
|
+
phase: 'parsing',
|
|
273
|
+
current,
|
|
274
|
+
total,
|
|
275
|
+
currentFile: files[Math.min(current, files.length - 1)],
|
|
276
|
+
message: `Parsed ${current}/${total} files`,
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
// Phase 3: Storing
|
|
280
|
+
onProgress?.({
|
|
281
|
+
phase: 'storing',
|
|
282
|
+
current: 0,
|
|
283
|
+
total: parseResults.length,
|
|
284
|
+
message: 'Storing parsed data in knowledge graph...',
|
|
285
|
+
});
|
|
286
|
+
for (let i = 0; i < parseResults.length; i++) {
|
|
287
|
+
const result = parseResults[i];
|
|
288
|
+
// If any single file fails, log and continue
|
|
289
|
+
try {
|
|
290
|
+
if (result.nodes.length === 0 && result.parseErrors.length > 0) {
|
|
291
|
+
stats.filesSkipped++;
|
|
292
|
+
stats.parseErrors += result.parseErrors.length;
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
stats.filesParsed++;
|
|
296
|
+
stats.nodesCreated += result.nodes.length;
|
|
297
|
+
stats.edgesCreated += result.edges.length;
|
|
298
|
+
// Track language distribution
|
|
299
|
+
if (result.language !== 'unknown') {
|
|
300
|
+
stats.languages[result.language] = (stats.languages[result.language] ?? 0) + 1;
|
|
296
301
|
}
|
|
302
|
+
if (result.parseErrors.length > 0) {
|
|
303
|
+
stats.parseErrors += result.parseErrors.length;
|
|
304
|
+
}
|
|
305
|
+
// Store in graph
|
|
306
|
+
this.graph.replaceFileData(result.filePath, result.nodes, result.edges);
|
|
307
|
+
if (i % 50 === 0) {
|
|
308
|
+
onProgress?.({
|
|
309
|
+
phase: 'storing',
|
|
310
|
+
current: i + 1,
|
|
311
|
+
total: parseResults.length,
|
|
312
|
+
currentFile: result.filePath,
|
|
313
|
+
message: `Stored ${i + 1}/${parseResults.length} files`,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
// Memory pressure check every 100 files
|
|
317
|
+
if (i % 100 === 0 && i > 0) {
|
|
318
|
+
const mem = process.memoryUsage();
|
|
319
|
+
if (mem.heapUsed / mem.heapTotal > MEMORY_PRESSURE_THRESHOLD) {
|
|
320
|
+
process.stderr.write(`⚠ Memory pressure during fullIndex at file ${i}/${parseResults.length}. Stopping early.\n`);
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
stats.parseErrors++;
|
|
297
327
|
}
|
|
298
328
|
}
|
|
329
|
+
// Cleanup orphaned edges after full reindex
|
|
330
|
+
try {
|
|
331
|
+
this.graph.cleanOrphanedEdges();
|
|
332
|
+
}
|
|
299
333
|
catch {
|
|
300
|
-
|
|
334
|
+
// Non-critical cleanup
|
|
301
335
|
}
|
|
336
|
+
// Phase 4: Complete
|
|
337
|
+
stats.durationMs = Date.now() - startTime;
|
|
338
|
+
onProgress?.({
|
|
339
|
+
phase: 'complete',
|
|
340
|
+
current: stats.filesParsed,
|
|
341
|
+
total: stats.filesScanned,
|
|
342
|
+
message: `Indexing complete: ${stats.filesParsed} files, ${stats.nodesCreated} nodes, ${stats.edgesCreated} edges in ${stats.durationMs}ms`,
|
|
343
|
+
});
|
|
344
|
+
return stats;
|
|
345
|
+
}
|
|
346
|
+
finally {
|
|
347
|
+
this.watcher?.resume();
|
|
348
|
+
release();
|
|
302
349
|
}
|
|
303
|
-
// Phase 4: Complete
|
|
304
|
-
stats.durationMs = Date.now() - startTime;
|
|
305
|
-
onProgress?.({
|
|
306
|
-
phase: 'complete',
|
|
307
|
-
current: stats.filesParsed,
|
|
308
|
-
total: stats.filesScanned,
|
|
309
|
-
message: `Indexing complete: ${stats.filesParsed} files, ${stats.nodesCreated} nodes, ${stats.edgesCreated} edges in ${stats.durationMs}ms`,
|
|
310
|
-
});
|
|
311
|
-
return stats;
|
|
312
350
|
}
|
|
313
351
|
/**
|
|
314
352
|
* Perform an incremental index — only re-parse files that have changed.
|
|
@@ -320,186 +358,192 @@ export class Indexer {
|
|
|
320
358
|
* @returns Indexing statistics
|
|
321
359
|
*/
|
|
322
360
|
async incrementalIndex(onProgress) {
|
|
323
|
-
const
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
if (
|
|
361
|
+
const release = await this.acquireIndexLock();
|
|
362
|
+
try {
|
|
363
|
+
const startTime = Date.now();
|
|
364
|
+
const stats = {
|
|
365
|
+
filesScanned: 0,
|
|
366
|
+
filesParsed: 0,
|
|
367
|
+
filesSkipped: 0,
|
|
368
|
+
filesDeleted: 0,
|
|
369
|
+
nodesCreated: 0,
|
|
370
|
+
edgesCreated: 0,
|
|
371
|
+
parseErrors: 0,
|
|
372
|
+
durationMs: 0,
|
|
373
|
+
languages: {},
|
|
374
|
+
};
|
|
375
|
+
// Phase 1: Scanning
|
|
376
|
+
onProgress?.({
|
|
377
|
+
phase: 'scanning',
|
|
378
|
+
current: 0,
|
|
379
|
+
total: 0,
|
|
380
|
+
message: 'Scanning for changes...',
|
|
381
|
+
});
|
|
382
|
+
const currentFiles = await this.scanFiles();
|
|
383
|
+
stats.filesScanned = currentFiles.length;
|
|
384
|
+
const indexedFiles = new Set(this.graph.getIndexedFiles());
|
|
385
|
+
const currentFileSet = new Set(currentFiles);
|
|
386
|
+
// Find files to add/update and files to delete
|
|
387
|
+
const filesToParse = [];
|
|
388
|
+
const filesToDelete = [];
|
|
389
|
+
// ── mtime-first staleness detection (10x faster) ──────────
|
|
390
|
+
// First check mtime+size via stat() (~0.1ms/file). Only hash
|
|
391
|
+
// files whose metadata changed (~3ms/file). This avoids
|
|
392
|
+
// reading every file on every incremental index.
|
|
393
|
+
for (const filePath of currentFiles) {
|
|
394
|
+
const existingHash = this.graph.getFileHash(filePath);
|
|
395
|
+
if (!existingHash) {
|
|
396
|
+
// New file — always parse
|
|
397
|
+
filesToParse.push(filePath);
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
// Fast path: check mtime+size from file_index table
|
|
401
|
+
const fileEntry = this.graph.getFileIndexEntry(filePath);
|
|
402
|
+
try {
|
|
403
|
+
const fileStat = await stat(filePath);
|
|
404
|
+
if (fileEntry) {
|
|
405
|
+
// If mtime AND size match, file is unchanged (fast path — 99% of cases)
|
|
406
|
+
if (fileStat.mtimeMs === fileEntry.mtime_ms && fileStat.size === fileEntry.size_bytes) {
|
|
407
|
+
stats.filesSkipped++;
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
// Slow path: metadata changed, verify with content hash
|
|
412
|
+
const content = await readFile(filePath, 'utf-8');
|
|
413
|
+
const currentHash = generateContentHash(content);
|
|
414
|
+
if (currentHash !== existingHash) {
|
|
415
|
+
filesToParse.push(filePath);
|
|
416
|
+
}
|
|
417
|
+
else {
|
|
418
|
+
// Content unchanged despite metadata change — update file_index
|
|
419
|
+
this.graph.upsertFileIndex(filePath, fileStat.mtimeMs, fileStat.size, currentHash);
|
|
367
420
|
stats.filesSkipped++;
|
|
368
|
-
continue;
|
|
369
421
|
}
|
|
370
422
|
}
|
|
371
|
-
|
|
372
|
-
const content = await readFile(filePath, 'utf-8');
|
|
373
|
-
const currentHash = generateContentHash(content);
|
|
374
|
-
if (currentHash !== existingHash) {
|
|
375
|
-
filesToParse.push(filePath);
|
|
376
|
-
}
|
|
377
|
-
else {
|
|
378
|
-
// Content unchanged despite metadata change — update file_index
|
|
379
|
-
this.graph.upsertFileIndex(filePath, fileStat.mtimeMs, fileStat.size, currentHash);
|
|
423
|
+
catch {
|
|
380
424
|
stats.filesSkipped++;
|
|
381
425
|
}
|
|
382
426
|
}
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
for (const indexedFile of indexedFiles) {
|
|
389
|
-
if (!currentFileSet.has(indexedFile)) {
|
|
390
|
-
filesToDelete.push(indexedFile);
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
// Phase 2: Handle deletions
|
|
394
|
-
if (filesToDelete.length > 0) {
|
|
395
|
-
onProgress?.({
|
|
396
|
-
phase: 'cleanup',
|
|
397
|
-
current: 0,
|
|
398
|
-
total: filesToDelete.length,
|
|
399
|
-
message: `Removing ${filesToDelete.length} deleted files...`,
|
|
400
|
-
});
|
|
401
|
-
for (const filePath of filesToDelete) {
|
|
402
|
-
this.graph.deleteFileNodes(filePath);
|
|
403
|
-
this.graph.removeFileIndex(filePath);
|
|
404
|
-
stats.filesDeleted++;
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
// Phase 3: Parse changed files
|
|
408
|
-
if (filesToParse.length > 0) {
|
|
409
|
-
onProgress?.({
|
|
410
|
-
phase: 'parsing',
|
|
411
|
-
current: 0,
|
|
412
|
-
total: filesToParse.length,
|
|
413
|
-
message: `Parsing ${filesToParse.length} changed files...`,
|
|
414
|
-
});
|
|
415
|
-
// ── Memory-aware indexing ─────────────────────────────
|
|
416
|
-
// Check memory pressure before parsing. If approaching
|
|
417
|
-
// 80% heap usage, stop and report partial results.
|
|
418
|
-
const memInfo = process.memoryUsage();
|
|
419
|
-
if (memInfo.heapUsed / memInfo.heapTotal > MEMORY_PRESSURE_THRESHOLD) {
|
|
420
|
-
console.error(`[ai-mind-map] Memory pressure: ${(memInfo.heapUsed / 1024 / 1024).toFixed(0)}MB / ` +
|
|
421
|
-
`${(memInfo.heapTotal / 1024 / 1024).toFixed(0)}MB. ` +
|
|
422
|
-
`Skipping ${filesToParse.length} files to prevent OOM.`);
|
|
423
|
-
stats.filesSkipped += filesToParse.length;
|
|
427
|
+
// Check for deleted files
|
|
428
|
+
for (const indexedFile of indexedFiles) {
|
|
429
|
+
if (!currentFileSet.has(indexedFile)) {
|
|
430
|
+
filesToDelete.push(indexedFile);
|
|
431
|
+
}
|
|
424
432
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
message: `Parsed ${current}/${total} changed files`,
|
|
433
|
-
});
|
|
433
|
+
// Phase 2: Handle deletions
|
|
434
|
+
if (filesToDelete.length > 0) {
|
|
435
|
+
onProgress?.({
|
|
436
|
+
phase: 'cleanup',
|
|
437
|
+
current: 0,
|
|
438
|
+
total: filesToDelete.length,
|
|
439
|
+
message: `Removing ${filesToDelete.length} deleted files...`,
|
|
434
440
|
});
|
|
435
|
-
|
|
441
|
+
for (const filePath of filesToDelete) {
|
|
442
|
+
this.graph.deleteFileNodes(filePath);
|
|
443
|
+
this.graph.removeFileIndex(filePath);
|
|
444
|
+
stats.filesDeleted++;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
// Phase 3: Parse changed files
|
|
448
|
+
if (filesToParse.length > 0) {
|
|
436
449
|
onProgress?.({
|
|
437
|
-
phase: '
|
|
450
|
+
phase: 'parsing',
|
|
438
451
|
current: 0,
|
|
439
|
-
total:
|
|
440
|
-
message:
|
|
452
|
+
total: filesToParse.length,
|
|
453
|
+
message: `Parsing ${filesToParse.length} changed files...`,
|
|
441
454
|
});
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
455
|
+
// ── Memory-aware indexing ─────────────────────────────
|
|
456
|
+
// Check memory pressure before parsing. If approaching
|
|
457
|
+
// 80% heap usage, stop and report partial results.
|
|
458
|
+
const memInfo = process.memoryUsage();
|
|
459
|
+
if (memInfo.heapUsed / memInfo.heapTotal > MEMORY_PRESSURE_THRESHOLD) {
|
|
460
|
+
console.error(`[ai-mind-map] Memory pressure: ${(memInfo.heapUsed / 1024 / 1024).toFixed(0)}MB / ` +
|
|
461
|
+
`${(memInfo.heapTotal / 1024 / 1024).toFixed(0)}MB. ` +
|
|
462
|
+
`Skipping ${filesToParse.length} files to prevent OOM.`);
|
|
463
|
+
stats.filesSkipped += filesToParse.length;
|
|
464
|
+
}
|
|
465
|
+
else {
|
|
466
|
+
const parseResults = await parseFiles(filesToParse, 8, (current, total) => {
|
|
467
|
+
onProgress?.({
|
|
468
|
+
phase: 'parsing',
|
|
469
|
+
current,
|
|
470
|
+
total,
|
|
471
|
+
currentFile: filesToParse[Math.min(current, filesToParse.length - 1)],
|
|
472
|
+
message: `Parsed ${current}/${total} changed files`,
|
|
473
|
+
});
|
|
474
|
+
});
|
|
475
|
+
// Phase 4: Store results
|
|
476
|
+
onProgress?.({
|
|
477
|
+
phase: 'storing',
|
|
478
|
+
current: 0,
|
|
479
|
+
total: parseResults.length,
|
|
480
|
+
message: 'Updating knowledge graph...',
|
|
481
|
+
});
|
|
482
|
+
for (let i = 0; i < parseResults.length; i++) {
|
|
483
|
+
// Check memory between files
|
|
484
|
+
if (i > 0 && i % 100 === 0) {
|
|
485
|
+
const mem = process.memoryUsage();
|
|
486
|
+
if (mem.heapUsed / mem.heapTotal > MEMORY_PRESSURE_THRESHOLD) {
|
|
487
|
+
console.error(`[ai-mind-map] Memory pressure at file ${i}/${parseResults.length}. ` +
|
|
488
|
+
`Stopping early to prevent OOM.`);
|
|
489
|
+
stats.filesSkipped += parseResults.length - i;
|
|
490
|
+
break;
|
|
491
|
+
}
|
|
451
492
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
493
|
+
const result = parseResults[i];
|
|
494
|
+
if (result.nodes.length === 0 && result.parseErrors.length > 0) {
|
|
495
|
+
stats.parseErrors += result.parseErrors.length;
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
stats.filesParsed++;
|
|
499
|
+
stats.nodesCreated += result.nodes.length;
|
|
500
|
+
stats.edgesCreated += result.edges.length;
|
|
501
|
+
if (result.language !== 'unknown') {
|
|
502
|
+
stats.languages[result.language] = (stats.languages[result.language] ?? 0) + 1;
|
|
503
|
+
}
|
|
504
|
+
if (result.parseErrors.length > 0) {
|
|
505
|
+
stats.parseErrors += result.parseErrors.length;
|
|
506
|
+
}
|
|
507
|
+
// Record changes before replacing (changelog diffing)
|
|
508
|
+
if (this.changelog) {
|
|
509
|
+
try {
|
|
510
|
+
const oldNodes = this.graph.getNodesForFile(result.filePath);
|
|
511
|
+
this.changelog.recordChanges(result.filePath, oldNodes, result.nodes);
|
|
512
|
+
}
|
|
513
|
+
catch {
|
|
514
|
+
// Changelog recording is non-critical
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
// Replace file data atomically
|
|
518
|
+
this.graph.replaceFileData(result.filePath, result.nodes, result.edges);
|
|
519
|
+
// Update file_index for future mtime-first detection
|
|
469
520
|
try {
|
|
470
|
-
const
|
|
471
|
-
|
|
521
|
+
const fileStat = await stat(result.filePath);
|
|
522
|
+
const fileHash = result.nodes.find(n => n.type === 'file')?.hash ?? '';
|
|
523
|
+
this.graph.upsertFileIndex(result.filePath, fileStat.mtimeMs, fileStat.size, fileHash);
|
|
472
524
|
}
|
|
473
525
|
catch {
|
|
474
|
-
//
|
|
526
|
+
// File may have been deleted between parse and store
|
|
475
527
|
}
|
|
476
528
|
}
|
|
477
|
-
// Replace file data atomically
|
|
478
|
-
this.graph.replaceFileData(result.filePath, result.nodes, result.edges);
|
|
479
|
-
// Update file_index for future mtime-first detection
|
|
480
|
-
try {
|
|
481
|
-
const fileStat = await stat(result.filePath);
|
|
482
|
-
const fileHash = result.nodes.find(n => n.type === 'file')?.hash ?? '';
|
|
483
|
-
this.graph.upsertFileIndex(result.filePath, fileStat.mtimeMs, fileStat.size, fileHash);
|
|
484
|
-
}
|
|
485
|
-
catch {
|
|
486
|
-
// File may have been deleted between parse and store
|
|
487
|
-
}
|
|
488
529
|
}
|
|
489
530
|
}
|
|
531
|
+
// Phase 5: Complete
|
|
532
|
+
stats.durationMs = Date.now() - startTime;
|
|
533
|
+
const changeCount = filesToParse.length + filesToDelete.length;
|
|
534
|
+
onProgress?.({
|
|
535
|
+
phase: 'complete',
|
|
536
|
+
current: changeCount,
|
|
537
|
+
total: stats.filesScanned,
|
|
538
|
+
message: changeCount > 0
|
|
539
|
+
? `Incremental index: ${stats.filesParsed} updated, ${stats.filesDeleted} deleted in ${stats.durationMs}ms`
|
|
540
|
+
: `No changes detected (${stats.filesScanned} files checked in ${stats.durationMs}ms)`,
|
|
541
|
+
});
|
|
542
|
+
return stats;
|
|
543
|
+
}
|
|
544
|
+
finally {
|
|
545
|
+
release();
|
|
490
546
|
}
|
|
491
|
-
// Phase 5: Complete
|
|
492
|
-
stats.durationMs = Date.now() - startTime;
|
|
493
|
-
const changeCount = filesToParse.length + filesToDelete.length;
|
|
494
|
-
onProgress?.({
|
|
495
|
-
phase: 'complete',
|
|
496
|
-
current: changeCount,
|
|
497
|
-
total: stats.filesScanned,
|
|
498
|
-
message: changeCount > 0
|
|
499
|
-
? `Incremental index: ${stats.filesParsed} updated, ${stats.filesDeleted} deleted in ${stats.durationMs}ms`
|
|
500
|
-
: `No changes detected (${stats.filesScanned} files checked in ${stats.durationMs}ms)`,
|
|
501
|
-
});
|
|
502
|
-
return stats;
|
|
503
547
|
}
|
|
504
548
|
/**
|
|
505
549
|
* Index a single file (e.g., when file watcher detects a change).
|
|
@@ -508,46 +552,52 @@ export class Indexer {
|
|
|
508
552
|
* @returns Parse result, or null if file was skipped
|
|
509
553
|
*/
|
|
510
554
|
async indexFile(filePath) {
|
|
511
|
-
|
|
512
|
-
const fileProjectRoot = this.findProjectRootForFile(filePath) ?? this.config.projectRoot;
|
|
513
|
-
const fileIgnore = this.getIgnoreForFile(filePath);
|
|
514
|
-
// Check if file should be ignored using the correct project's rules
|
|
515
|
-
const relPath = relative(fileProjectRoot, filePath).replace(/\\/g, '/');
|
|
516
|
-
if (fileIgnore.ignores(relPath))
|
|
517
|
-
return null;
|
|
518
|
-
if (!isSupportedFile(filePath))
|
|
519
|
-
return null;
|
|
520
|
-
// Skip files that are too large (prevent OOM)
|
|
521
|
-
const fileStat = await stat(filePath).catch(() => null);
|
|
522
|
-
if (!fileStat || fileStat.size > MAX_FILE_SIZE_BYTES) {
|
|
523
|
-
return null;
|
|
524
|
-
}
|
|
555
|
+
const release = await this.acquireIndexLock();
|
|
525
556
|
try {
|
|
526
|
-
|
|
557
|
+
// Determine which project this file belongs to (multi-project support)
|
|
558
|
+
const fileProjectRoot = this.findProjectRootForFile(filePath) ?? this.config.projectRoot;
|
|
559
|
+
const fileIgnore = this.getIgnoreForFile(filePath);
|
|
560
|
+
// Check if file should be ignored using the correct project's rules
|
|
561
|
+
const relPath = relative(fileProjectRoot, filePath).replace(/\\/g, '/');
|
|
562
|
+
if (fileIgnore.ignores(relPath))
|
|
527
563
|
return null;
|
|
528
|
-
if (
|
|
564
|
+
if (!isSupportedFile(filePath))
|
|
529
565
|
return null;
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
566
|
+
// Skip files that are too large (prevent OOM)
|
|
567
|
+
const fileStat = await stat(filePath).catch(() => null);
|
|
568
|
+
if (!fileStat || fileStat.size > MAX_FILE_SIZE_BYTES) {
|
|
569
|
+
return null;
|
|
570
|
+
}
|
|
571
|
+
try {
|
|
572
|
+
if (fileStat.size > this.config.maxFileSize)
|
|
573
|
+
return null;
|
|
574
|
+
if (fileStat.size === 0)
|
|
575
|
+
return null;
|
|
576
|
+
}
|
|
577
|
+
catch {
|
|
578
|
+
// File may have been deleted
|
|
579
|
+
this.graph.deleteFileNodes(filePath);
|
|
580
|
+
return null;
|
|
581
|
+
}
|
|
582
|
+
const result = await parseFile(filePath);
|
|
583
|
+
if (result.nodes.length > 0) {
|
|
584
|
+
// Record changes before replacing (changelog diffing)
|
|
585
|
+
if (this.changelog) {
|
|
586
|
+
try {
|
|
587
|
+
const oldNodes = this.graph.getNodesForFile(filePath);
|
|
588
|
+
this.changelog.recordChanges(filePath, oldNodes, result.nodes);
|
|
589
|
+
}
|
|
590
|
+
catch {
|
|
591
|
+
// Changelog recording is non-critical
|
|
592
|
+
}
|
|
546
593
|
}
|
|
594
|
+
this.graph.replaceFileData(filePath, result.nodes, result.edges);
|
|
547
595
|
}
|
|
548
|
-
|
|
596
|
+
return result;
|
|
597
|
+
}
|
|
598
|
+
finally {
|
|
599
|
+
release();
|
|
549
600
|
}
|
|
550
|
-
return result;
|
|
551
601
|
}
|
|
552
602
|
/**
|
|
553
603
|
* Remove a file from the index (e.g., when file watcher detects deletion).
|