nano-brain 2026.6.8 → 2026.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/bench.ts +694 -1
- package/src/watcher.ts +6 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nano-brain",
|
|
3
|
-
"version": "2026.6.
|
|
3
|
+
"version": "2026.6.9",
|
|
4
4
|
"description": "Persistent memory and code intelligence for AI coding agents. Local MCP server with self-learning hybrid search (BM25 + vector + knowledge graph + LLM reranking), automatic session ingestion, codebase indexing, and 22 tools. Learns your preferences over time. Works with OpenCode, Claude, Cursor, Windsurf, and any MCP client.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/bench.ts
CHANGED
|
@@ -2,6 +2,8 @@ import { createStore, computeHash, indexDocument } from './store.js';
|
|
|
2
2
|
import { loadCollectionConfig } from './collections.js';
|
|
3
3
|
import { createEmbeddingProvider, checkOllamaHealth, detectOllamaUrl } from './embeddings.js';
|
|
4
4
|
import { hybridSearch } from './search.js';
|
|
5
|
+
import { traverse, getRelatedDocuments } from './connection-graph.js';
|
|
6
|
+
import { ConsolidationAgent } from './consolidation.js';
|
|
5
7
|
import type { GlobalOptions } from './index.js';
|
|
6
8
|
import type { Store } from './types.js';
|
|
7
9
|
import * as fs from 'fs';
|
|
@@ -37,6 +39,11 @@ const DEFAULT_ITERATIONS: Record<string, number> = {
|
|
|
37
39
|
embed: 5,
|
|
38
40
|
cache: 20,
|
|
39
41
|
store: 20,
|
|
42
|
+
connections: 10,
|
|
43
|
+
quality: 1,
|
|
44
|
+
scale: 1,
|
|
45
|
+
consolidation: 10,
|
|
46
|
+
memory: 1,
|
|
40
47
|
};
|
|
41
48
|
|
|
42
49
|
async function runBenchmark(
|
|
@@ -234,6 +241,672 @@ async function runStoreSuite(iterations: number, dbPath: string): Promise<BenchR
|
|
|
234
241
|
return results;
|
|
235
242
|
}
|
|
236
243
|
|
|
244
|
+
async function runConnectionsSuite(iterations: number): Promise<BenchResult[]> {
|
|
245
|
+
const results: BenchResult[] = [];
|
|
246
|
+
const tempDbPath = path.join(os.tmpdir(), `nano-brain-bench-conn-${Date.now()}.sqlite`);
|
|
247
|
+
const store = await createStore(tempDbPath);
|
|
248
|
+
|
|
249
|
+
for (let i = 0; i < 100; i++) {
|
|
250
|
+
const content = `# Doc ${i}\n\nContent for document ${i} in connections benchmark.`;
|
|
251
|
+
const hash = computeHash(content);
|
|
252
|
+
store.insertContent(hash, content);
|
|
253
|
+
store.insertDocument({
|
|
254
|
+
collection: 'bench',
|
|
255
|
+
path: `/bench/conn-doc-${i}.md`,
|
|
256
|
+
title: `Connection Doc ${i}`,
|
|
257
|
+
hash,
|
|
258
|
+
createdAt: new Date().toISOString(),
|
|
259
|
+
modifiedAt: new Date().toISOString(),
|
|
260
|
+
active: true,
|
|
261
|
+
projectHash: 'bench',
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
results.push(
|
|
266
|
+
await runBenchmark('insertConnection (single)', () => {
|
|
267
|
+
store.insertConnection({
|
|
268
|
+
fromDocId: 1,
|
|
269
|
+
toDocId: 2,
|
|
270
|
+
relationshipType: 'related',
|
|
271
|
+
description: null,
|
|
272
|
+
strength: 0.8,
|
|
273
|
+
createdBy: 'user',
|
|
274
|
+
projectHash: 'bench',
|
|
275
|
+
});
|
|
276
|
+
}, iterations)
|
|
277
|
+
);
|
|
278
|
+
|
|
279
|
+
let batchCounter = 0;
|
|
280
|
+
results.push(
|
|
281
|
+
await runBenchmark('insertConnection (batch 100)', () => {
|
|
282
|
+
for (let i = 0; i < 100; i++) {
|
|
283
|
+
store.insertConnection({
|
|
284
|
+
fromDocId: (batchCounter * 100 + i) % 100 + 1,
|
|
285
|
+
toDocId: (batchCounter * 100 + i + 1) % 100 + 1,
|
|
286
|
+
relationshipType: 'related',
|
|
287
|
+
description: null,
|
|
288
|
+
strength: 0.7,
|
|
289
|
+
createdBy: 'user',
|
|
290
|
+
projectHash: 'bench',
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
batchCounter++;
|
|
294
|
+
}, iterations)
|
|
295
|
+
);
|
|
296
|
+
|
|
297
|
+
const doc0Conns = store.getConnectionsForDocument(100);
|
|
298
|
+
results.push(
|
|
299
|
+
await runBenchmark('getConnections (0 conns)', () => {
|
|
300
|
+
store.getConnectionsForDocument(100);
|
|
301
|
+
}, iterations)
|
|
302
|
+
);
|
|
303
|
+
|
|
304
|
+
for (let i = 0; i < 10; i++) {
|
|
305
|
+
store.insertConnection({
|
|
306
|
+
fromDocId: 50,
|
|
307
|
+
toDocId: 60 + i,
|
|
308
|
+
relationshipType: 'supports',
|
|
309
|
+
description: null,
|
|
310
|
+
strength: 0.9,
|
|
311
|
+
createdBy: 'user',
|
|
312
|
+
projectHash: 'bench',
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
results.push(
|
|
316
|
+
await runBenchmark('getConnections (10 conns)', () => {
|
|
317
|
+
store.getConnectionsForDocument(50);
|
|
318
|
+
}, iterations)
|
|
319
|
+
);
|
|
320
|
+
|
|
321
|
+
for (let i = 0; i < 40; i++) {
|
|
322
|
+
store.insertConnection({
|
|
323
|
+
fromDocId: 30,
|
|
324
|
+
toDocId: (i % 99) + 1,
|
|
325
|
+
relationshipType: 'extends',
|
|
326
|
+
description: null,
|
|
327
|
+
strength: 0.85,
|
|
328
|
+
createdBy: 'user',
|
|
329
|
+
projectHash: 'bench',
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
results.push(
|
|
333
|
+
await runBenchmark('getConnections (50 conns)', () => {
|
|
334
|
+
store.getConnectionsForDocument(30);
|
|
335
|
+
}, iterations)
|
|
336
|
+
);
|
|
337
|
+
|
|
338
|
+
results.push(
|
|
339
|
+
await runBenchmark('traverse depth=1', () => {
|
|
340
|
+
traverse(store, 1, { maxDepth: 1 });
|
|
341
|
+
}, iterations)
|
|
342
|
+
);
|
|
343
|
+
|
|
344
|
+
results.push(
|
|
345
|
+
await runBenchmark('traverse depth=2', () => {
|
|
346
|
+
traverse(store, 1, { maxDepth: 2 });
|
|
347
|
+
}, iterations)
|
|
348
|
+
);
|
|
349
|
+
|
|
350
|
+
results.push(
|
|
351
|
+
await runBenchmark('traverse depth=3', () => {
|
|
352
|
+
traverse(store, 1, { maxDepth: 3 });
|
|
353
|
+
}, iterations)
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
results.push(
|
|
357
|
+
await runBenchmark('getConnectionCount', () => {
|
|
358
|
+
store.getConnectionCount(1);
|
|
359
|
+
}, iterations)
|
|
360
|
+
);
|
|
361
|
+
|
|
362
|
+
const connToDelete = store.insertConnection({
|
|
363
|
+
fromDocId: 99,
|
|
364
|
+
toDocId: 98,
|
|
365
|
+
relationshipType: 'related',
|
|
366
|
+
description: null,
|
|
367
|
+
strength: 0.5,
|
|
368
|
+
createdBy: 'user',
|
|
369
|
+
projectHash: 'bench',
|
|
370
|
+
});
|
|
371
|
+
let deleteId = connToDelete;
|
|
372
|
+
results.push(
|
|
373
|
+
await runBenchmark('deleteConnection', () => {
|
|
374
|
+
store.deleteConnection(deleteId);
|
|
375
|
+
deleteId = store.insertConnection({
|
|
376
|
+
fromDocId: 99,
|
|
377
|
+
toDocId: 98,
|
|
378
|
+
relationshipType: 'related',
|
|
379
|
+
description: null,
|
|
380
|
+
strength: 0.5,
|
|
381
|
+
createdBy: 'user',
|
|
382
|
+
projectHash: 'bench',
|
|
383
|
+
});
|
|
384
|
+
}, iterations)
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
store.close();
|
|
388
|
+
try {
|
|
389
|
+
fs.unlinkSync(tempDbPath);
|
|
390
|
+
fs.unlinkSync(tempDbPath + '-wal');
|
|
391
|
+
fs.unlinkSync(tempDbPath + '-shm');
|
|
392
|
+
} catch {
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
return results;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async function runQualitySuite(
|
|
399
|
+
store: Store,
|
|
400
|
+
embedder: { embed(text: string): Promise<{ embedding: number[] }> } | null,
|
|
401
|
+
iterations: number
|
|
402
|
+
): Promise<BenchResult[]> {
|
|
403
|
+
const results: BenchResult[] = [];
|
|
404
|
+
const tempDbPath = path.join(os.tmpdir(), `nano-brain-bench-quality-${Date.now()}.sqlite`);
|
|
405
|
+
const tempStore = await createStore(tempDbPath);
|
|
406
|
+
|
|
407
|
+
const topics = [
|
|
408
|
+
{ name: 'authentication', keywords: ['JWT', 'token', 'login', 'session', 'OAuth', 'password', 'auth middleware', 'bearer', 'refresh token', 'credentials'] },
|
|
409
|
+
{ name: 'database-optimization', keywords: ['index', 'query plan', 'slow query', 'N+1', 'connection pool', 'transaction', 'deadlock', 'vacuum', 'explain analyze', 'cache hit'] },
|
|
410
|
+
{ name: 'error-handling', keywords: ['try catch', 'exception', 'error boundary', 'stack trace', 'retry logic', 'circuit breaker', 'fallback', 'graceful degradation', 'error code', 'validation'] },
|
|
411
|
+
{ name: 'deployment', keywords: ['Docker', 'Kubernetes', 'CI/CD', 'rolling update', 'blue-green', 'canary', 'helm chart', 'container', 'orchestration', 'scaling'] },
|
|
412
|
+
{ name: 'testing', keywords: ['unit test', 'integration test', 'mock', 'stub', 'fixture', 'coverage', 'assertion', 'test runner', 'snapshot', 'e2e'] },
|
|
413
|
+
];
|
|
414
|
+
|
|
415
|
+
const docIndices: Record<string, number[]> = {};
|
|
416
|
+
let docId = 0;
|
|
417
|
+
for (const topic of topics) {
|
|
418
|
+
docIndices[topic.name] = [];
|
|
419
|
+
for (let i = 0; i < 10; i++) {
|
|
420
|
+
const content = `# ${topic.name} Document ${i}\n\n${topic.keywords.slice(0, 5 + i % 5).join(', ')}.\n\nThis document covers ${topic.name} concepts including ${topic.keywords[i % topic.keywords.length]}.`;
|
|
421
|
+
const hash = computeHash(content + docId);
|
|
422
|
+
tempStore.insertContent(hash, content);
|
|
423
|
+
tempStore.insertDocument({
|
|
424
|
+
collection: 'bench',
|
|
425
|
+
path: `/bench/${topic.name}-${i}.md`,
|
|
426
|
+
title: `${topic.name} ${i}`,
|
|
427
|
+
hash,
|
|
428
|
+
createdAt: new Date().toISOString(),
|
|
429
|
+
modifiedAt: new Date().toISOString(),
|
|
430
|
+
active: true,
|
|
431
|
+
projectHash: 'bench',
|
|
432
|
+
});
|
|
433
|
+
docIndices[topic.name].push(docId);
|
|
434
|
+
docId++;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const queries = [
|
|
439
|
+
{ query: 'JWT token authentication middleware', relevant: docIndices['authentication'] },
|
|
440
|
+
{ query: 'database query optimization index', relevant: docIndices['database-optimization'] },
|
|
441
|
+
{ query: 'error handling try catch exception', relevant: docIndices['error-handling'] },
|
|
442
|
+
{ query: 'Docker Kubernetes deployment', relevant: docIndices['deployment'] },
|
|
443
|
+
{ query: 'unit test mock coverage', relevant: docIndices['testing'] },
|
|
444
|
+
{ query: 'OAuth login session credentials', relevant: docIndices['authentication'] },
|
|
445
|
+
{ query: 'slow query N+1 connection pool', relevant: docIndices['database-optimization'] },
|
|
446
|
+
{ query: 'circuit breaker retry fallback', relevant: docIndices['error-handling'] },
|
|
447
|
+
{ query: 'CI/CD rolling update canary', relevant: docIndices['deployment'] },
|
|
448
|
+
{ query: 'integration test fixture assertion', relevant: docIndices['testing'] },
|
|
449
|
+
];
|
|
450
|
+
|
|
451
|
+
let ftsPrecision = 0, ftsRecall = 0, ftsMrr = 0;
|
|
452
|
+
for (const q of queries) {
|
|
453
|
+
const ftsResults = tempStore.searchFTS(q.query, { limit: 10 });
|
|
454
|
+
const topIds = ftsResults.map(r => Number(r.id));
|
|
455
|
+
const relevantSet = new Set(q.relevant);
|
|
456
|
+
|
|
457
|
+
const top5Relevant = topIds.slice(0, 5).filter((id: number) => relevantSet.has(id)).length;
|
|
458
|
+
ftsPrecision += top5Relevant / 5;
|
|
459
|
+
|
|
460
|
+
const top10Relevant = topIds.filter((id: number) => relevantSet.has(id)).length;
|
|
461
|
+
ftsRecall += top10Relevant / q.relevant.length;
|
|
462
|
+
|
|
463
|
+
const firstRelevantRank = topIds.findIndex((id: number) => relevantSet.has(id));
|
|
464
|
+
ftsMrr += firstRelevantRank >= 0 ? 1 / (firstRelevantRank + 1) : 0;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
results.push({
|
|
468
|
+
name: 'P@5 (FTS)',
|
|
469
|
+
iterations: 1,
|
|
470
|
+
meanMs: ftsPrecision / queries.length,
|
|
471
|
+
minMs: 0,
|
|
472
|
+
maxMs: 0,
|
|
473
|
+
opsPerSec: 0,
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
results.push({
|
|
477
|
+
name: 'Recall@10 (FTS)',
|
|
478
|
+
iterations: 1,
|
|
479
|
+
meanMs: ftsRecall / queries.length,
|
|
480
|
+
minMs: 0,
|
|
481
|
+
maxMs: 0,
|
|
482
|
+
opsPerSec: 0,
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
results.push({
|
|
486
|
+
name: 'MRR (FTS)',
|
|
487
|
+
iterations: 1,
|
|
488
|
+
meanMs: ftsMrr / queries.length,
|
|
489
|
+
minMs: 0,
|
|
490
|
+
maxMs: 0,
|
|
491
|
+
opsPerSec: 0,
|
|
492
|
+
});
|
|
493
|
+
|
|
494
|
+
if (embedder) {
|
|
495
|
+
let hybridPrecision = 0, hybridRecall = 0, hybridMrr = 0;
|
|
496
|
+
for (const q of queries) {
|
|
497
|
+
const hybridResults = await hybridSearch(tempStore, { query: q.query, limit: 10 }, { embedder });
|
|
498
|
+
const topIds = hybridResults.map((r: any) => Number(r.id));
|
|
499
|
+
const relevantSet = new Set(q.relevant);
|
|
500
|
+
|
|
501
|
+
const top5Relevant = topIds.slice(0, 5).filter((id: number) => relevantSet.has(id)).length;
|
|
502
|
+
hybridPrecision += top5Relevant / 5;
|
|
503
|
+
|
|
504
|
+
const top10Relevant = topIds.filter((id: number) => relevantSet.has(id)).length;
|
|
505
|
+
hybridRecall += top10Relevant / q.relevant.length;
|
|
506
|
+
|
|
507
|
+
const firstRelevantRank = topIds.findIndex((id: number) => relevantSet.has(id));
|
|
508
|
+
hybridMrr += firstRelevantRank >= 0 ? 1 / (firstRelevantRank + 1) : 0;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
results.push({
|
|
512
|
+
name: 'P@5 (Hybrid)',
|
|
513
|
+
iterations: 1,
|
|
514
|
+
meanMs: hybridPrecision / queries.length,
|
|
515
|
+
minMs: 0,
|
|
516
|
+
maxMs: 0,
|
|
517
|
+
opsPerSec: 0,
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
results.push({
|
|
521
|
+
name: 'Recall@10 (Hybrid)',
|
|
522
|
+
iterations: 1,
|
|
523
|
+
meanMs: hybridRecall / queries.length,
|
|
524
|
+
minMs: 0,
|
|
525
|
+
maxMs: 0,
|
|
526
|
+
opsPerSec: 0,
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
results.push({
|
|
530
|
+
name: 'MRR (Hybrid)',
|
|
531
|
+
iterations: 1,
|
|
532
|
+
meanMs: hybridMrr / queries.length,
|
|
533
|
+
minMs: 0,
|
|
534
|
+
maxMs: 0,
|
|
535
|
+
opsPerSec: 0,
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
tempStore.close();
|
|
540
|
+
try {
|
|
541
|
+
fs.unlinkSync(tempDbPath);
|
|
542
|
+
fs.unlinkSync(tempDbPath + '-wal');
|
|
543
|
+
fs.unlinkSync(tempDbPath + '-shm');
|
|
544
|
+
} catch {
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
return results;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
async function runScaleSuite(iterations: number): Promise<BenchResult[]> {
|
|
551
|
+
const results: BenchResult[] = [];
|
|
552
|
+
const scalePoints = [100, 500, 1000, 5000];
|
|
553
|
+
|
|
554
|
+
for (const n of scalePoints) {
|
|
555
|
+
const tempDbPath = path.join(os.tmpdir(), `nano-brain-bench-scale-${n}-${Date.now()}.sqlite`);
|
|
556
|
+
const store = await createStore(tempDbPath);
|
|
557
|
+
|
|
558
|
+
const insertStart = performance.now();
|
|
559
|
+
for (let i = 0; i < n; i++) {
|
|
560
|
+
const content = `# Scale Doc ${i}\n\nContent for scale testing at ${n} documents. Keywords: test benchmark performance scale.`;
|
|
561
|
+
const hash = computeHash(content + i + n);
|
|
562
|
+
store.insertContent(hash, content);
|
|
563
|
+
store.insertDocument({
|
|
564
|
+
collection: 'bench',
|
|
565
|
+
path: `/bench/scale-${n}-${i}.md`,
|
|
566
|
+
title: `Scale Doc ${i}`,
|
|
567
|
+
hash,
|
|
568
|
+
createdAt: new Date().toISOString(),
|
|
569
|
+
modifiedAt: new Date().toISOString(),
|
|
570
|
+
active: true,
|
|
571
|
+
projectHash: 'bench',
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
const insertEnd = performance.now();
|
|
575
|
+
const insertMs = insertEnd - insertStart;
|
|
576
|
+
|
|
577
|
+
results.push({
|
|
578
|
+
name: `insert @${n}`,
|
|
579
|
+
iterations: 1,
|
|
580
|
+
meanMs: insertMs,
|
|
581
|
+
minMs: insertMs,
|
|
582
|
+
maxMs: insertMs,
|
|
583
|
+
opsPerSec: n / (insertMs / 1000),
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
const searchTimes: number[] = [];
|
|
587
|
+
for (let i = 0; i < 5; i++) {
|
|
588
|
+
const start = performance.now();
|
|
589
|
+
store.searchFTS('benchmark performance', { limit: 10 });
|
|
590
|
+
searchTimes.push(performance.now() - start);
|
|
591
|
+
}
|
|
592
|
+
const avgSearchMs = searchTimes.reduce((a, b) => a + b, 0) / searchTimes.length;
|
|
593
|
+
|
|
594
|
+
results.push({
|
|
595
|
+
name: `FTS search @${n}`,
|
|
596
|
+
iterations: 5,
|
|
597
|
+
meanMs: avgSearchMs,
|
|
598
|
+
minMs: Math.min(...searchTimes),
|
|
599
|
+
maxMs: Math.max(...searchTimes),
|
|
600
|
+
opsPerSec: 1000 / avgSearchMs,
|
|
601
|
+
});
|
|
602
|
+
|
|
603
|
+
const healthTimes: number[] = [];
|
|
604
|
+
for (let i = 0; i < 5; i++) {
|
|
605
|
+
const start = performance.now();
|
|
606
|
+
store.getIndexHealth();
|
|
607
|
+
healthTimes.push(performance.now() - start);
|
|
608
|
+
}
|
|
609
|
+
const avgHealthMs = healthTimes.reduce((a, b) => a + b, 0) / healthTimes.length;
|
|
610
|
+
|
|
611
|
+
results.push({
|
|
612
|
+
name: `indexHealth @${n}`,
|
|
613
|
+
iterations: 5,
|
|
614
|
+
meanMs: avgHealthMs,
|
|
615
|
+
minMs: Math.min(...healthTimes),
|
|
616
|
+
maxMs: Math.max(...healthTimes),
|
|
617
|
+
opsPerSec: 1000 / avgHealthMs,
|
|
618
|
+
});
|
|
619
|
+
|
|
620
|
+
const rss = process.memoryUsage().rss / (1024 * 1024);
|
|
621
|
+
results.push({
|
|
622
|
+
name: `RSS @${n}`,
|
|
623
|
+
iterations: 1,
|
|
624
|
+
meanMs: rss,
|
|
625
|
+
minMs: rss,
|
|
626
|
+
maxMs: rss,
|
|
627
|
+
opsPerSec: 0,
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
store.close();
|
|
631
|
+
try {
|
|
632
|
+
fs.unlinkSync(tempDbPath);
|
|
633
|
+
fs.unlinkSync(tempDbPath + '-wal');
|
|
634
|
+
fs.unlinkSync(tempDbPath + '-shm');
|
|
635
|
+
} catch {
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
return results;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
async function runConsolidationSuite(iterations: number): Promise<BenchResult[]> {
|
|
643
|
+
const results: BenchResult[] = [];
|
|
644
|
+
const tempDbPath = path.join(os.tmpdir(), `nano-brain-bench-consol-${Date.now()}.sqlite`);
|
|
645
|
+
const store = await createStore(tempDbPath);
|
|
646
|
+
|
|
647
|
+
for (let i = 0; i < 50; i++) {
|
|
648
|
+
const content = `# Memory ${i}\n\nThis is a memory document for consolidation benchmarking. Topic: ${i % 5 === 0 ? 'authentication' : i % 5 === 1 ? 'database' : i % 5 === 2 ? 'errors' : i % 5 === 3 ? 'deployment' : 'testing'}.`;
|
|
649
|
+
const hash = computeHash(content + i);
|
|
650
|
+
store.insertContent(hash, content);
|
|
651
|
+
store.insertDocument({
|
|
652
|
+
collection: 'memory',
|
|
653
|
+
path: `/memory/consol-${i}.md`,
|
|
654
|
+
title: `Memory ${i}`,
|
|
655
|
+
hash,
|
|
656
|
+
createdAt: new Date().toISOString(),
|
|
657
|
+
modifiedAt: new Date().toISOString(),
|
|
658
|
+
active: true,
|
|
659
|
+
projectHash: 'bench',
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
const mockLlmResponse = JSON.stringify([{
|
|
664
|
+
sourceIds: [1, 2, 3],
|
|
665
|
+
summary: 'test summary',
|
|
666
|
+
insight: 'test insight',
|
|
667
|
+
connections: [{ fromId: 1, toId: 2, relationship: 'supports', confidence: 0.9 }],
|
|
668
|
+
overallConfidence: 0.85,
|
|
669
|
+
}]);
|
|
670
|
+
|
|
671
|
+
const mockLlmProvider = {
|
|
672
|
+
complete: async (_prompt: string) => ({ text: mockLlmResponse, tokensUsed: 100 }),
|
|
673
|
+
model: 'mock',
|
|
674
|
+
};
|
|
675
|
+
|
|
676
|
+
const agent = new ConsolidationAgent(store, { llmProvider: mockLlmProvider });
|
|
677
|
+
|
|
678
|
+
const db = store.getDb();
|
|
679
|
+
|
|
680
|
+
results.push(
|
|
681
|
+
await runBenchmark('getUnconsolidatedMemories', () => {
|
|
682
|
+
db.prepare(`
|
|
683
|
+
SELECT d.id, d.title, d.path, d.hash, c.body
|
|
684
|
+
FROM documents d
|
|
685
|
+
JOIN content c ON d.hash = c.hash
|
|
686
|
+
WHERE d.collection = 'memory'
|
|
687
|
+
AND d.active = 1
|
|
688
|
+
AND d.superseded_by IS NULL
|
|
689
|
+
ORDER BY d.modified_at DESC
|
|
690
|
+
LIMIT 20
|
|
691
|
+
`).all();
|
|
692
|
+
}, iterations)
|
|
693
|
+
);
|
|
694
|
+
|
|
695
|
+
const sampleMemories = [
|
|
696
|
+
{ id: 1, title: 'Memory 1', path: '/memory/1.md', hash: 'h1', body: 'Sample body content for memory 1' },
|
|
697
|
+
{ id: 2, title: 'Memory 2', path: '/memory/2.md', hash: 'h2', body: 'Sample body content for memory 2' },
|
|
698
|
+
{ id: 3, title: 'Memory 3', path: '/memory/3.md', hash: 'h3', body: 'Sample body content for memory 3' },
|
|
699
|
+
];
|
|
700
|
+
|
|
701
|
+
results.push(
|
|
702
|
+
await runBenchmark('buildConsolidationPrompt', () => {
|
|
703
|
+
const prompt = `You are a memory consolidation agent. Analyze the following memories and find connections between them.
|
|
704
|
+
|
|
705
|
+
For each group of related memories, output a JSON object with:
|
|
706
|
+
- sourceIds: array of memory IDs that are related
|
|
707
|
+
- summary: a concise summary of the related memories
|
|
708
|
+
- insight: a new insight derived from connecting these memories
|
|
709
|
+
- connections: array of {fromId, toId, relationship, confidence} objects
|
|
710
|
+
- overallConfidence: 0.0-1.0 rating of how confident you are in this consolidation
|
|
711
|
+
|
|
712
|
+
Output a JSON array of consolidation objects. Only include consolidations with confidence >= 0.7.
|
|
713
|
+
|
|
714
|
+
Memories:
|
|
715
|
+
${sampleMemories.map(m => `[ID: ${m.id}] ${m.title}\n${m.body.substring(0, 500)}`).join('\n\n---\n\n')}
|
|
716
|
+
|
|
717
|
+
Respond with ONLY a JSON array, no other text.`;
|
|
718
|
+
}, iterations)
|
|
719
|
+
);
|
|
720
|
+
|
|
721
|
+
results.push(
|
|
722
|
+
await runBenchmark('parseConsolidationResponse', () => {
|
|
723
|
+
const text = mockLlmResponse;
|
|
724
|
+
const jsonMatch = text.match(/\[[\s\S]*\]/);
|
|
725
|
+
if (jsonMatch) {
|
|
726
|
+
JSON.parse(jsonMatch[0]);
|
|
727
|
+
}
|
|
728
|
+
}, iterations)
|
|
729
|
+
);
|
|
730
|
+
|
|
731
|
+
results.push(
|
|
732
|
+
await runBenchmark('applyConsolidation', () => {
|
|
733
|
+
const result = {
|
|
734
|
+
sourceIds: [1, 2, 3],
|
|
735
|
+
summary: 'test summary',
|
|
736
|
+
insight: 'test insight',
|
|
737
|
+
connections: [] as Array<{ fromId: number; toId: number; relationship: string; confidence: number }>,
|
|
738
|
+
overallConfidence: 0.85,
|
|
739
|
+
};
|
|
740
|
+
db.prepare(`
|
|
741
|
+
INSERT INTO consolidations (source_ids, summary, insight, connections, confidence, created_at)
|
|
742
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
743
|
+
`).run(
|
|
744
|
+
JSON.stringify(result.sourceIds),
|
|
745
|
+
result.summary,
|
|
746
|
+
result.insight,
|
|
747
|
+
JSON.stringify(result.connections),
|
|
748
|
+
result.overallConfidence,
|
|
749
|
+
new Date().toISOString()
|
|
750
|
+
);
|
|
751
|
+
}, iterations)
|
|
752
|
+
);
|
|
753
|
+
|
|
754
|
+
store.close();
|
|
755
|
+
try {
|
|
756
|
+
fs.unlinkSync(tempDbPath);
|
|
757
|
+
fs.unlinkSync(tempDbPath + '-wal');
|
|
758
|
+
fs.unlinkSync(tempDbPath + '-shm');
|
|
759
|
+
} catch {
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
return results;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
async function runMemorySuite(iterations: number): Promise<BenchResult[]> {
|
|
766
|
+
const results: BenchResult[] = [];
|
|
767
|
+
|
|
768
|
+
const baselineRss = process.memoryUsage().rss / (1024 * 1024);
|
|
769
|
+
results.push({
|
|
770
|
+
name: 'baseline RSS',
|
|
771
|
+
iterations: 1,
|
|
772
|
+
meanMs: baselineRss,
|
|
773
|
+
minMs: baselineRss,
|
|
774
|
+
maxMs: baselineRss,
|
|
775
|
+
opsPerSec: 0,
|
|
776
|
+
});
|
|
777
|
+
|
|
778
|
+
const tempDbPath = path.join(os.tmpdir(), `nano-brain-bench-mem-${Date.now()}.sqlite`);
|
|
779
|
+
const store = await createStore(tempDbPath);
|
|
780
|
+
|
|
781
|
+
const insertDocs = async (count: number) => {
|
|
782
|
+
for (let i = 0; i < count; i++) {
|
|
783
|
+
const content = `# Memory Doc ${i}\n\nContent for memory benchmarking document ${i}. This is filler text to simulate real document sizes.`;
|
|
784
|
+
const hash = computeHash(content + Date.now() + i);
|
|
785
|
+
store.insertContent(hash, content);
|
|
786
|
+
store.insertDocument({
|
|
787
|
+
collection: 'bench',
|
|
788
|
+
path: `/bench/mem-${Date.now()}-${i}.md`,
|
|
789
|
+
title: `Memory Doc ${i}`,
|
|
790
|
+
hash,
|
|
791
|
+
createdAt: new Date().toISOString(),
|
|
792
|
+
modifiedAt: new Date().toISOString(),
|
|
793
|
+
active: true,
|
|
794
|
+
projectHash: 'bench',
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
};
|
|
798
|
+
|
|
799
|
+
await insertDocs(100);
|
|
800
|
+
const rss100 = process.memoryUsage().rss / (1024 * 1024);
|
|
801
|
+
results.push({
|
|
802
|
+
name: 'RSS after 100 docs',
|
|
803
|
+
iterations: 1,
|
|
804
|
+
meanMs: rss100,
|
|
805
|
+
minMs: rss100,
|
|
806
|
+
maxMs: rss100,
|
|
807
|
+
opsPerSec: 0,
|
|
808
|
+
});
|
|
809
|
+
|
|
810
|
+
await insertDocs(400);
|
|
811
|
+
const rss500 = process.memoryUsage().rss / (1024 * 1024);
|
|
812
|
+
results.push({
|
|
813
|
+
name: 'RSS after 500 docs',
|
|
814
|
+
iterations: 1,
|
|
815
|
+
meanMs: rss500,
|
|
816
|
+
minMs: rss500,
|
|
817
|
+
maxMs: rss500,
|
|
818
|
+
opsPerSec: 0,
|
|
819
|
+
});
|
|
820
|
+
|
|
821
|
+
await insertDocs(500);
|
|
822
|
+
const rss1000 = process.memoryUsage().rss / (1024 * 1024);
|
|
823
|
+
results.push({
|
|
824
|
+
name: 'RSS after 1000 docs',
|
|
825
|
+
iterations: 1,
|
|
826
|
+
meanMs: rss1000,
|
|
827
|
+
minMs: rss1000,
|
|
828
|
+
maxMs: rss1000,
|
|
829
|
+
opsPerSec: 0,
|
|
830
|
+
});
|
|
831
|
+
|
|
832
|
+
for (let i = 0; i < 100; i++) {
|
|
833
|
+
store.insertConnection({
|
|
834
|
+
fromDocId: (i % 100) + 1,
|
|
835
|
+
toDocId: ((i + 1) % 100) + 1,
|
|
836
|
+
relationshipType: 'related',
|
|
837
|
+
description: null,
|
|
838
|
+
strength: 0.8,
|
|
839
|
+
createdBy: 'user',
|
|
840
|
+
projectHash: 'bench',
|
|
841
|
+
});
|
|
842
|
+
}
|
|
843
|
+
const rss100Conns = process.memoryUsage().rss / (1024 * 1024);
|
|
844
|
+
results.push({
|
|
845
|
+
name: 'RSS after 100 conns',
|
|
846
|
+
iterations: 1,
|
|
847
|
+
meanMs: rss100Conns,
|
|
848
|
+
minMs: rss100Conns,
|
|
849
|
+
maxMs: rss100Conns,
|
|
850
|
+
opsPerSec: 0,
|
|
851
|
+
});
|
|
852
|
+
|
|
853
|
+
for (let i = 0; i < 400; i++) {
|
|
854
|
+
store.insertConnection({
|
|
855
|
+
fromDocId: (i % 100) + 1,
|
|
856
|
+
toDocId: ((i + 50) % 100) + 1,
|
|
857
|
+
relationshipType: 'supports',
|
|
858
|
+
description: null,
|
|
859
|
+
strength: 0.7,
|
|
860
|
+
createdBy: 'user',
|
|
861
|
+
projectHash: 'bench',
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
const rss500Conns = process.memoryUsage().rss / (1024 * 1024);
|
|
865
|
+
results.push({
|
|
866
|
+
name: 'RSS after 500 conns',
|
|
867
|
+
iterations: 1,
|
|
868
|
+
meanMs: rss500Conns,
|
|
869
|
+
minMs: rss500Conns,
|
|
870
|
+
maxMs: rss500Conns,
|
|
871
|
+
opsPerSec: 0,
|
|
872
|
+
});
|
|
873
|
+
|
|
874
|
+
const searchPromises = [];
|
|
875
|
+
for (let i = 0; i < 10; i++) {
|
|
876
|
+
searchPromises.push(Promise.resolve(store.searchFTS('memory benchmarking', { limit: 10 })));
|
|
877
|
+
}
|
|
878
|
+
await Promise.all(searchPromises);
|
|
879
|
+
const peakRss = process.memoryUsage().rss / (1024 * 1024);
|
|
880
|
+
results.push({
|
|
881
|
+
name: 'peak RSS',
|
|
882
|
+
iterations: 1,
|
|
883
|
+
meanMs: peakRss,
|
|
884
|
+
minMs: peakRss,
|
|
885
|
+
maxMs: peakRss,
|
|
886
|
+
opsPerSec: 0,
|
|
887
|
+
});
|
|
888
|
+
|
|
889
|
+
const heapGrowth = ((rss1000 - rss100) / 900) * 1024;
|
|
890
|
+
results.push({
|
|
891
|
+
name: 'heap growth rate (KB/doc)',
|
|
892
|
+
iterations: 1,
|
|
893
|
+
meanMs: heapGrowth,
|
|
894
|
+
minMs: heapGrowth,
|
|
895
|
+
maxMs: heapGrowth,
|
|
896
|
+
opsPerSec: 0,
|
|
897
|
+
});
|
|
898
|
+
|
|
899
|
+
store.close();
|
|
900
|
+
try {
|
|
901
|
+
fs.unlinkSync(tempDbPath);
|
|
902
|
+
fs.unlinkSync(tempDbPath + '-wal');
|
|
903
|
+
fs.unlinkSync(tempDbPath + '-shm');
|
|
904
|
+
} catch {
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
return results;
|
|
908
|
+
}
|
|
909
|
+
|
|
237
910
|
function formatHumanReadable(suiteResults: SuiteResults): string {
|
|
238
911
|
const lines: string[] = [];
|
|
239
912
|
lines.push('nano-brain Benchmark Results');
|
|
@@ -382,7 +1055,7 @@ export async function handleBench(globalOpts: GlobalOptions, commandArgs: string
|
|
|
382
1055
|
|
|
383
1056
|
const suitesToRun = options.suite
|
|
384
1057
|
? [options.suite]
|
|
385
|
-
: ['search', 'embed', 'cache', 'store'];
|
|
1058
|
+
: ['search', 'embed', 'cache', 'store', 'connections', 'quality', 'scale', 'consolidation', 'memory'];
|
|
386
1059
|
|
|
387
1060
|
for (const suite of suitesToRun) {
|
|
388
1061
|
const iterations = options.iterations || DEFAULT_ITERATIONS[suite] || 10;
|
|
@@ -408,6 +1081,26 @@ export async function handleBench(globalOpts: GlobalOptions, commandArgs: string
|
|
|
408
1081
|
suiteResults.store = await runStoreSuite(iterations, globalOpts.dbPath);
|
|
409
1082
|
break;
|
|
410
1083
|
|
|
1084
|
+
case 'connections':
|
|
1085
|
+
suiteResults.connections = await runConnectionsSuite(iterations);
|
|
1086
|
+
break;
|
|
1087
|
+
|
|
1088
|
+
case 'quality':
|
|
1089
|
+
suiteResults.quality = await runQualitySuite(store, embedder, iterations);
|
|
1090
|
+
break;
|
|
1091
|
+
|
|
1092
|
+
case 'scale':
|
|
1093
|
+
suiteResults.scale = await runScaleSuite(iterations);
|
|
1094
|
+
break;
|
|
1095
|
+
|
|
1096
|
+
case 'consolidation':
|
|
1097
|
+
suiteResults.consolidation = await runConsolidationSuite(iterations);
|
|
1098
|
+
break;
|
|
1099
|
+
|
|
1100
|
+
case 'memory':
|
|
1101
|
+
suiteResults.memory = await runMemorySuite(iterations);
|
|
1102
|
+
break;
|
|
1103
|
+
|
|
411
1104
|
default:
|
|
412
1105
|
console.error(`Unknown suite: ${suite}`);
|
|
413
1106
|
}
|
package/src/watcher.ts
CHANGED
|
@@ -337,13 +337,12 @@ export function startWatcher(options: WatcherOptions): Watcher {
|
|
|
337
337
|
watchedPaths.add(expandedPath)
|
|
338
338
|
}
|
|
339
339
|
}
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
}
|
|
340
|
+
// NOTE: We intentionally do NOT watch the workspace root for codebase changes.
|
|
341
|
+
// Large workspaces (e.g. 30+ subprojects, 8000+ dirs) exhaust OS file descriptor
|
|
342
|
+
// limits even with node_modules excluded. Codebase changes are picked up by the
|
|
343
|
+
// poll-based reindex cycle instead (pollIntervalMs, default 5min).
|
|
344
|
+
if (codebaseConfig?.enabled) {
|
|
345
|
+
log('watcher', 'Codebase watching uses poll-based reindex (not fs.watch) to avoid EMFILE on large workspaces')
|
|
347
346
|
}
|
|
348
347
|
const deduped: string[] = []
|
|
349
348
|
for (const p of pathsToWatch) {
|