@nxuss/lemma 1.36.0 → 1.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/README.md +88 -426
  2. package/bin/brain-ingest.js +22 -0
  3. package/bin/doctor.js +31 -4
  4. package/bin/init.js +92 -51
  5. package/dashboard/README.md +48 -19
  6. package/dashboard/dist/assets/index-CLXWgzTm.css +1 -0
  7. package/dashboard/dist/assets/index-CoMIMxyC.js +451 -0
  8. package/dashboard/dist/index.html +2 -2
  9. package/dist/cjs/api/dashboardRoutes.d.ts.map +1 -1
  10. package/dist/cjs/api/dashboardRoutes.js +7 -3
  11. package/dist/cjs/api/dashboardRoutes.js.map +1 -1
  12. package/dist/cjs/autopilot/Notifier.d.ts.map +1 -1
  13. package/dist/cjs/autopilot/Notifier.js +3 -0
  14. package/dist/cjs/autopilot/Notifier.js.map +1 -1
  15. package/dist/cjs/cli/hostHooks.d.ts +13 -1
  16. package/dist/cjs/cli/hostHooks.d.ts.map +1 -1
  17. package/dist/cjs/cli/hostHooks.js +34 -4
  18. package/dist/cjs/cli/hostHooks.js.map +1 -1
  19. package/dist/cjs/cli/lemma-proxy.d.ts.map +1 -1
  20. package/dist/cjs/cli/lemma-proxy.js +235 -107
  21. package/dist/cjs/cli/lemma-proxy.js.map +1 -1
  22. package/dist/cjs/mcp/index.js +42 -18
  23. package/dist/cjs/mcp/index.js.map +1 -1
  24. package/dist/cjs/mcp/prompts.d.ts +1 -1
  25. package/dist/cjs/mcp/prompts.d.ts.map +1 -1
  26. package/dist/cjs/mcp/prompts.js +2 -0
  27. package/dist/cjs/mcp/prompts.js.map +1 -1
  28. package/dist/cjs/mcp/resources.d.ts.map +1 -1
  29. package/dist/cjs/mcp/resources.js +23 -4
  30. package/dist/cjs/mcp/resources.js.map +1 -1
  31. package/dist/cjs/mcp/tools/memory.js +11 -11
  32. package/dist/cjs/mcp/tools/memory.js.map +1 -1
  33. package/dist/cjs/mcp/tools/misc.js +1 -1
  34. package/dist/cjs/mcp/tools/misc.js.map +1 -1
  35. package/dist/cjs/mcp/tools/review.js +1 -1
  36. package/dist/cjs/mcp/tools/review.js.map +1 -1
  37. package/dist/cjs/mcp/tools/workspace.d.ts.map +1 -1
  38. package/dist/cjs/mcp/tools/workspace.js +10 -7
  39. package/dist/cjs/mcp/tools/workspace.js.map +1 -1
  40. package/dist/cjs/mcp/tools.d.ts +16 -1
  41. package/dist/cjs/mcp/tools.d.ts.map +1 -1
  42. package/dist/cjs/mcp/tools.js +63 -33
  43. package/dist/cjs/mcp/tools.js.map +1 -1
  44. package/dist/cjs/mcp/workspaceShared.d.ts.map +1 -1
  45. package/dist/cjs/mcp/workspaceShared.js +6 -9
  46. package/dist/cjs/mcp/workspaceShared.js.map +1 -1
  47. package/dist/cjs/proxy/ProjectRegistry.d.ts +24 -0
  48. package/dist/cjs/proxy/ProjectRegistry.d.ts.map +1 -1
  49. package/dist/cjs/proxy/ProjectRegistry.js +50 -3
  50. package/dist/cjs/proxy/ProjectRegistry.js.map +1 -1
  51. package/dist/cjs/subconscious/HostRecall.d.ts +10 -1
  52. package/dist/cjs/subconscious/HostRecall.d.ts.map +1 -1
  53. package/dist/cjs/subconscious/HostRecall.js +68 -6
  54. package/dist/cjs/subconscious/HostRecall.js.map +1 -1
  55. package/dist/cjs/subconscious/recallDigest.d.ts +34 -0
  56. package/dist/cjs/subconscious/recallDigest.d.ts.map +1 -0
  57. package/dist/cjs/subconscious/recallDigest.js +82 -0
  58. package/dist/cjs/subconscious/recallDigest.js.map +1 -0
  59. package/dist/cjs/utils/ProjectIdentity.d.ts +8 -0
  60. package/dist/cjs/utils/ProjectIdentity.d.ts.map +1 -1
  61. package/dist/cjs/utils/ProjectIdentity.js +15 -0
  62. package/dist/cjs/utils/ProjectIdentity.js.map +1 -1
  63. package/dist/cjs/utils/SavingsLedger.d.ts +2 -0
  64. package/dist/cjs/utils/SavingsLedger.d.ts.map +1 -1
  65. package/dist/cjs/utils/SavingsLedger.js +5 -5
  66. package/dist/cjs/utils/SavingsLedger.js.map +1 -1
  67. package/dist/cjs/utils/TokenReceipt.d.ts +2 -0
  68. package/dist/cjs/utils/TokenReceipt.d.ts.map +1 -1
  69. package/dist/cjs/utils/TokenReceipt.js +11 -1
  70. package/dist/cjs/utils/TokenReceipt.js.map +1 -1
  71. package/dist/cjs/utils/reportSavings.d.ts +7 -0
  72. package/dist/cjs/utils/reportSavings.d.ts.map +1 -1
  73. package/dist/cjs/utils/reportSavings.js +14 -2
  74. package/dist/cjs/utils/reportSavings.js.map +1 -1
  75. package/dist/esm/api/dashboardRoutes.d.ts.map +1 -1
  76. package/dist/esm/api/dashboardRoutes.js +7 -3
  77. package/dist/esm/api/dashboardRoutes.js.map +1 -1
  78. package/dist/esm/autopilot/Notifier.d.ts.map +1 -1
  79. package/dist/esm/autopilot/Notifier.js +3 -0
  80. package/dist/esm/autopilot/Notifier.js.map +1 -1
  81. package/dist/esm/cli/hostHooks.d.ts +13 -1
  82. package/dist/esm/cli/hostHooks.d.ts.map +1 -1
  83. package/dist/esm/cli/hostHooks.js +32 -3
  84. package/dist/esm/cli/hostHooks.js.map +1 -1
  85. package/dist/esm/cli/lemma-proxy.d.ts.map +1 -1
  86. package/dist/esm/cli/lemma-proxy.js +236 -108
  87. package/dist/esm/cli/lemma-proxy.js.map +1 -1
  88. package/dist/esm/mcp/index.js +42 -18
  89. package/dist/esm/mcp/index.js.map +1 -1
  90. package/dist/esm/mcp/prompts.d.ts +1 -1
  91. package/dist/esm/mcp/prompts.d.ts.map +1 -1
  92. package/dist/esm/mcp/prompts.js +2 -0
  93. package/dist/esm/mcp/prompts.js.map +1 -1
  94. package/dist/esm/mcp/resources.d.ts.map +1 -1
  95. package/dist/esm/mcp/resources.js +23 -4
  96. package/dist/esm/mcp/resources.js.map +1 -1
  97. package/dist/esm/mcp/tools/memory.js +11 -11
  98. package/dist/esm/mcp/tools/memory.js.map +1 -1
  99. package/dist/esm/mcp/tools/misc.js +1 -1
  100. package/dist/esm/mcp/tools/misc.js.map +1 -1
  101. package/dist/esm/mcp/tools/review.js +1 -1
  102. package/dist/esm/mcp/tools/review.js.map +1 -1
  103. package/dist/esm/mcp/tools/workspace.d.ts.map +1 -1
  104. package/dist/esm/mcp/tools/workspace.js +10 -7
  105. package/dist/esm/mcp/tools/workspace.js.map +1 -1
  106. package/dist/esm/mcp/tools.d.ts +16 -1
  107. package/dist/esm/mcp/tools.d.ts.map +1 -1
  108. package/dist/esm/mcp/tools.js +62 -34
  109. package/dist/esm/mcp/tools.js.map +1 -1
  110. package/dist/esm/mcp/workspaceShared.d.ts.map +1 -1
  111. package/dist/esm/mcp/workspaceShared.js +6 -9
  112. package/dist/esm/mcp/workspaceShared.js.map +1 -1
  113. package/dist/esm/proxy/ProjectRegistry.d.ts +24 -0
  114. package/dist/esm/proxy/ProjectRegistry.d.ts.map +1 -1
  115. package/dist/esm/proxy/ProjectRegistry.js +49 -2
  116. package/dist/esm/proxy/ProjectRegistry.js.map +1 -1
  117. package/dist/esm/subconscious/HostRecall.d.ts +10 -1
  118. package/dist/esm/subconscious/HostRecall.d.ts.map +1 -1
  119. package/dist/esm/subconscious/HostRecall.js +66 -5
  120. package/dist/esm/subconscious/HostRecall.js.map +1 -1
  121. package/dist/esm/subconscious/recallDigest.d.ts +34 -0
  122. package/dist/esm/subconscious/recallDigest.d.ts.map +1 -0
  123. package/dist/esm/subconscious/recallDigest.js +76 -0
  124. package/dist/esm/subconscious/recallDigest.js.map +1 -0
  125. package/dist/esm/utils/ProjectIdentity.d.ts +8 -0
  126. package/dist/esm/utils/ProjectIdentity.d.ts.map +1 -1
  127. package/dist/esm/utils/ProjectIdentity.js +14 -0
  128. package/dist/esm/utils/ProjectIdentity.js.map +1 -1
  129. package/dist/esm/utils/SavingsLedger.d.ts +2 -0
  130. package/dist/esm/utils/SavingsLedger.d.ts.map +1 -1
  131. package/dist/esm/utils/SavingsLedger.js +1 -1
  132. package/dist/esm/utils/SavingsLedger.js.map +1 -1
  133. package/dist/esm/utils/TokenReceipt.d.ts +2 -0
  134. package/dist/esm/utils/TokenReceipt.d.ts.map +1 -1
  135. package/dist/esm/utils/TokenReceipt.js +10 -1
  136. package/dist/esm/utils/TokenReceipt.js.map +1 -1
  137. package/dist/esm/utils/reportSavings.d.ts +7 -0
  138. package/dist/esm/utils/reportSavings.d.ts.map +1 -1
  139. package/dist/esm/utils/reportSavings.js +13 -2
  140. package/dist/esm/utils/reportSavings.js.map +1 -1
  141. package/package.json +10 -3
  142. package/dashboard/dist/assets/index-D6nem4OA.js +0 -461
  143. package/dashboard/dist/assets/index-DNz43ews.css +0 -1
@@ -61,6 +61,8 @@ const ProjectStore_1 = require("../proxy/ProjectStore");
61
61
  const gitHooks_1 = require("./gitHooks");
62
62
  const hostHooks_1 = require("./hostHooks");
63
63
  const TheBrainV2_1 = require("../subconscious/TheBrainV2");
64
+ const utils_1 = require("../mcp/utils");
65
+ const ProjectIdentity_2 = require("../utils/ProjectIdentity");
64
66
  const chroma = new chromadb_1.ChromaClient({ host: 'localhost', port: 8000 });
65
67
  const dummyEmbeddingFunction = { generate: (_texts) => Promise.resolve([]) };
66
68
  const complexityRouter = new ComplexityRouter_1.default();
@@ -96,7 +98,7 @@ function writeJson(filePath, data) {
96
98
  catch { }
97
99
  }
98
100
  function getCollectionName(projectName) {
99
- return `lemma-cache-${projectName}`;
101
+ return (0, ProjectIdentity_2.cacheCollectionName)(projectName);
100
102
  }
101
103
  // detectProject/isProjectMismatch live in ../utils/ProjectIdentity — a side-effect-free
102
104
  // module shared with src/mcp/tools.ts, so the client and this daemon can never define
@@ -396,6 +398,11 @@ enabled = true
396
398
  }
397
399
  const EVENT_LOG = [];
398
400
  const MAX_EVENTS = 100;
401
+ /**
402
+ * Secrets masked since this daemon started. Counting 'privacy:mask' entries in EVENT_LOG only
403
+ * ever saw the last 100 events, and counted requests rather than secrets.
404
+ */
405
+ const SECRETS_MASKED = { count: 0 };
399
406
  // MCP Status tracking
400
407
  const MCP_STATUS = {
401
408
  lastActivity: Date.now(),
@@ -482,7 +489,7 @@ function startBackgroundClipboardWatcher(cliOpts) {
482
489
  }
483
490
  }
484
491
  catch { }
485
- const shouldWatch = cliOpts?.clipboard !== false && !configDisabled;
492
+ const shouldWatch = cliOpts?.clipboard !== false && !configDisabled && process.env.LEMMA_DISABLE_CLIPBOARD !== 'true';
486
493
  if (!shouldWatch) {
487
494
  console.log(`⏭️ [Clipboard] Background clipboard watcher is disabled (via options or config).`);
488
495
  return;
@@ -590,6 +597,20 @@ async function loadStats() {
590
597
  * Tokens a cache hit actually avoided regenerating, measured from the cached payload
591
598
  * that was served instead of an LLM call. Replaces the old hardcoded 2000 guess.
592
599
  */
600
+ /** The answer text inside a cached provider response (OpenAI, Anthropic or Gemini shape). */
601
+ function extractResponseText(data) {
602
+ const text = data?.choices?.[0]?.message?.content ??
603
+ data?.content?.find?.((c) => c?.type === 'text')?.text ??
604
+ data?.candidates?.[0]?.content?.parts?.[0]?.text;
605
+ if (typeof text === 'string')
606
+ return text;
607
+ try {
608
+ return JSON.stringify(data);
609
+ }
610
+ catch {
611
+ return '';
612
+ }
613
+ }
593
614
  function measureCachedTokens(data) {
594
615
  try {
595
616
  const chars = JSON.stringify(data)?.length;
@@ -611,6 +632,33 @@ function measureCachedTokens(data) {
611
632
  * round-trip), so a miss costs latency, not tokens. Charging it a fabricated token
612
633
  * figure would repeat exactly the error the cost ledger exists to correct.
613
634
  */
635
+ /**
636
+ * Machine-wide totals. Completions are attributed per project (see handleCompletion), and the
637
+ * dashboard — like the savings ledger — reports the whole daemon, not one project's slice.
638
+ */
639
+ function aggregateStats(stats) {
640
+ const sum = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0 };
641
+ for (const s of Object.values(stats || {})) {
642
+ // Stats written before /api/savings-record stopped touching them counted every MCP saving
643
+ // as a completion under provider 'mcp', with a mock 15ms latency. Those counts are exact in
644
+ // the provider breakdown, so take them back out; only real completions remain.
645
+ const mcp = s?.providers?.mcp || { hits: 0, misses: 0 };
646
+ const mcpCalls = (mcp.hits || 0) + (mcp.misses || 0);
647
+ sum.total += Math.max(0, (s?.total || 0) - mcpCalls);
648
+ sum.hits += Math.max(0, (s?.hits || 0) - (mcp.hits || 0));
649
+ sum.misses += Math.max(0, (s?.misses || 0) - (mcp.misses || 0));
650
+ sum.totalLatency += Math.max(0, (s?.totalLatency || 0) - mcpCalls * 15);
651
+ sum.totalTokensSaved += s?.totalTokensSaved || 0;
652
+ }
653
+ return sum;
654
+ }
655
+ /** Every semantic-cache collection the daemon writes: one per project, all "lemma-cache" prefixed. */
656
+ async function listCacheCollections() {
657
+ const cols = await chroma.listCollections();
658
+ return cols
659
+ .map((c) => (typeof c === 'string' ? c : c?.name))
660
+ .filter((n) => typeof n === 'string' && n.startsWith('lemma-cache'));
661
+ }
614
662
  async function recordStat(stats, project, fromCache, latencyMs, provider, tokensSaved) {
615
663
  if (!stats[project]) {
616
664
  stats[project] = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0, providers: {} };
@@ -622,7 +670,8 @@ async function recordStat(stats, project, fromCache, latencyMs, provider, tokens
622
670
  s.hits++;
623
671
  else
624
672
  s.misses++;
625
- const tkSaved = fromCache ? (tokensSaved || 2000) : (tokensSaved || 0);
673
+ // Unmeasured is 0, not a guess: a 2000 placeholder used to surface as "2000 tokens saved".
674
+ const tkSaved = tokensSaved || 0;
626
675
  s.totalTokensSaved += tkSaved;
627
676
  if (!s.providers[provider])
628
677
  s.providers[provider] = { hits: 0, misses: 0 };
@@ -630,9 +679,7 @@ async function recordStat(stats, project, fromCache, latencyMs, provider, tokens
630
679
  s.providers[provider].hits++;
631
680
  else
632
681
  s.providers[provider].misses++;
633
- // Only measured savings reach the ledger. `tokensSaved === undefined` means the
634
- // caller never counted, so tkSaved is the 2000 placeholder — usable for the
635
- // dashboard, not for a net-position claim.
682
+ // Only measured savings reach the ledger.
636
683
  if (fromCache && typeof tokensSaved === 'number' && tokensSaved > 0) {
637
684
  SavingsLedger_1.savingsLedger.recordTokens('cache', tokensSaved);
638
685
  }
@@ -659,6 +706,9 @@ function cacheGet(provider, prompt) {
659
706
  CACHE.delete(k);
660
707
  return null;
661
708
  }
709
+ // The only caller is the exact-hit path, so a returned entry is a served hit.
710
+ e.hits++;
711
+ e.lastHitAt = Date.now();
662
712
  return e;
663
713
  }
664
714
  function cacheSet(provider, prompt, data) {
@@ -670,7 +720,7 @@ function cacheSet(provider, prompt, data) {
670
720
  if (oldestKey)
671
721
  CACHE.delete(oldestKey);
672
722
  }
673
- CACHE.set(k, { input: prompt, data, createdAt: Date.now(), ttl: TTL_MS, provider });
723
+ CACHE.set(k, { input: prompt, data, createdAt: Date.now(), ttl: TTL_MS, provider, hits: 0 });
674
724
  }
675
725
  // ── Semantic Search (Pro) ──────────────────────────────────────────────────────
676
726
  async function getEmbedding(text) {
@@ -1056,18 +1106,33 @@ class LemmaServer {
1056
1106
  console.log('▶️ [Debugger] Resumed');
1057
1107
  res.json({ resumed: true, paused: false, timestamp: new Date().toISOString() });
1058
1108
  });
1059
- // Project registration from MCP handshake
1109
+ // Project registration from MCP handshake; type "mcp_heartbeat" is the 30s keep-alive
1060
1110
  this.app.post('/api/handshake', (req, res) => {
1061
- const { projectPath, projectName, stack } = req.body;
1111
+ const { projectPath, projectName, type } = req.body;
1112
+ // MCP servers send techStack as an array; older callers sent stack as a string.
1113
+ const rawStack = req.body.techStack ?? req.body.stack;
1114
+ const stack = Array.isArray(rawStack) ? rawStack.join(',') : typeof rawStack === 'string' ? rawStack : '';
1062
1115
  if (projectPath && projectName) {
1063
- this.registry.register(projectPath, projectName, stack || '');
1064
- console.log(`📋 [Handshake] Project registered: "${projectName}" (${projectPath})`);
1116
+ if (type === 'mcp_heartbeat') {
1117
+ const known = !!this.registry.get(projectPath);
1118
+ this.registry.heartbeat(projectPath, projectName, stack);
1119
+ if (!known)
1120
+ console.log(`📋 [Handshake] Project re-registered by heartbeat: "${projectName}" (${projectPath})`);
1121
+ }
1122
+ else {
1123
+ this.registry.register(projectPath, projectName, stack);
1124
+ console.log(`📋 [Handshake] Project registered: "${projectName}" (${projectPath})`);
1125
+ }
1065
1126
  res.json({ ok: true });
1066
1127
  }
1067
1128
  else {
1068
1129
  res.status(400).json({ error: 'projectPath and projectName required' });
1069
1130
  }
1070
1131
  });
1132
+ // Every project with an MCP server connected to this daemon, idle ones included
1133
+ this.app.get('/api/projects', (req, res) => {
1134
+ res.json({ projects: this.registry.snapshot() });
1135
+ });
1071
1136
  this.app.get('/v1/models', (req, res) => {
1072
1137
  res.json({ object: 'list', data: [
1073
1138
  { id: 'gpt-4o', object: 'model', owned_by: 'lemma-proxy' },
@@ -1096,10 +1161,10 @@ class LemmaServer {
1096
1161
  if (fs_1.default.existsSync(this.dashboardPath)) {
1097
1162
  this.app.use('/dashboard/', express_1.default.static(this.dashboardPath));
1098
1163
  this.app.get('/api/metrics', async (req, res) => {
1099
- const stats = this.stats[this.projectName] || { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0 };
1164
+ const stats = aggregateStats(this.stats);
1100
1165
  const hitRate = stats.total > 0 ? stats.hits / stats.total : 0;
1101
1166
  const avgLat = stats.total > 0 ? stats.totalLatency / stats.total : 0;
1102
- const secretsMasked = EVENT_LOG.filter(e => e.type === 'privacy:mask').length;
1167
+ const secretsMasked = SECRETS_MASKED.count;
1103
1168
  const ledgerSnap = SavingsLedger_1.savingsLedger.getSnapshot();
1104
1169
  let activeAgents = 1; // Always include the Gateway itself
1105
1170
  let totalAgents = 1;
@@ -1130,8 +1195,7 @@ class LemmaServer {
1130
1195
  });
1131
1196
  this.app.get('/api/agents', async (req, res) => {
1132
1197
  // Load statistics for the Gateway itself
1133
- const stats = await loadStats();
1134
- const statsProj = stats[this.projectName] || { total: 0, hits: 0 };
1198
+ const statsProj = aggregateStats(await loadStats());
1135
1199
  const total = statsProj.total || 0;
1136
1200
  const hits = statsProj.hits || 0;
1137
1201
  const gatewayAgent = {
@@ -1168,7 +1232,8 @@ class LemmaServer {
1168
1232
  type: e.type, // e.g., 'cache:hit' or 'cache:miss' which client normalizes to 'cache_hit' or 'cache_miss'
1169
1233
  latency: e.latency,
1170
1234
  tokensSaved: e.tokens || e.tokensImpact || 0,
1171
- costSaved: (e.tokens || e.tokensImpact || 0) ? ((e.tokens || e.tokensImpact || 0) / 1000) * 0.002 : 0,
1235
+ // Same per-token price as the savings ledger, so a row and the totals agree.
1236
+ costSaved: (e.tokens || e.tokensImpact || 0) * SavingsLedger_1.AVG_COST_PER_TOKEN,
1172
1237
  metadata: {
1173
1238
  provider: e.provider,
1174
1239
  project: e.project
@@ -1180,6 +1245,12 @@ class LemmaServer {
1180
1245
  eventData.args = e.args;
1181
1246
  eventData.result = e.result;
1182
1247
  eventData.tokensImpact = e.tokensImpact;
1248
+ eventData.error = e.error;
1249
+ eventData.receipt = e.receipt;
1250
+ eventData.projectPath = e.projectPath;
1251
+ // tokensImpact is the size of the tool result — a context cost, not a saving.
1252
+ eventData.tokensSaved = 0;
1253
+ eventData.costSaved = 0;
1183
1254
  }
1184
1255
  return eventData;
1185
1256
  });
@@ -1213,28 +1284,34 @@ class LemmaServer {
1213
1284
  this.app.get('/api/cache-stats', async (req, res) => {
1214
1285
  let topQueries = [];
1215
1286
  let totalCached = CACHE.size;
1216
- // Try to query ChromaDB for semantic cache count
1287
+ // Semantic entries across every project's collection (completions are cached per project)
1217
1288
  try {
1218
- const collection = await chroma.getOrCreateCollection({
1219
- name: getCollectionName(this.projectName),
1220
- embeddingFunction: dummyEmbeddingFunction,
1221
- metadata: { "hnsw:space": "cosine" }
1222
- });
1223
- const count = await collection.count();
1224
- totalCached += count;
1289
+ for (const name of await listCacheCollections()) {
1290
+ const collection = await chroma.getOrCreateCollection({
1291
+ name,
1292
+ embeddingFunction: dummyEmbeddingFunction,
1293
+ metadata: { "hnsw:space": "cosine" }
1294
+ });
1295
+ totalCached += await collection.count();
1296
+ }
1225
1297
  }
1226
1298
  catch { }
1227
- // Build top queries from in-memory CACHE
1299
+ // Most reused answers: per-entry hit counts and the measured size of the cached response.
1300
+ // These used to be a flat 2000 tokens at $2/M with the provider's hit count stamped on
1301
+ // every entry, and "last hit" was really the creation time.
1228
1302
  for (const [key, entry] of CACHE.entries()) {
1229
- const hitCount = EVENT_LOG.filter(e => e.type === 'cache:hit' && e.provider === entry.provider).length || 1;
1230
- const tokens = 2000;
1303
+ if (entry.hits === 0)
1304
+ continue;
1305
+ const tokensPerHit = measureCachedTokens(entry.data) || 0;
1306
+ const totalTokens = tokensPerHit * entry.hits;
1231
1307
  topQueries.push({
1232
1308
  queryHash: key.substring(0, 16),
1233
1309
  query: entry.input,
1234
- hitCount,
1235
- totalTokens: tokens * hitCount,
1236
- costSaved: (tokens * hitCount / 1000) * 0.002,
1237
- lastHit: new Date(entry.createdAt).toISOString()
1310
+ hitCount: entry.hits,
1311
+ totalTokens,
1312
+ costSaved: totalTokens * SavingsLedger_1.AVG_COST_PER_TOKEN,
1313
+ lastHit: new Date(entry.lastHitAt || entry.createdAt).toISOString(),
1314
+ responsePreview: extractResponseText(entry.data).slice(0, 600),
1238
1315
  });
1239
1316
  }
1240
1317
  // Try to query the stack orchestration API if running to append stack cache stats
@@ -1251,8 +1328,11 @@ class LemmaServer {
1251
1328
  catch { }
1252
1329
  // Sort by hitCount desc
1253
1330
  topQueries.sort((a, b) => b.hitCount - a.hitCount);
1254
- // Latency percentiles from EVENT_LOG
1255
- const latencies = EVENT_LOG.filter(e => e.latency).map(e => e.latency).sort((a, b) => a - b);
1331
+ // Latency percentiles of proxy completions only — MCP tool calls share EVENT_LOG.
1332
+ const latencies = EVENT_LOG
1333
+ .filter(e => (e.type === 'cache:hit' || e.type === 'cache:miss') && e.provider !== 'mcp' && typeof e.latency === 'number')
1334
+ .map(e => e.latency)
1335
+ .sort((a, b) => a - b);
1256
1336
  const latencyPercentiles = {
1257
1337
  p50: latencies[Math.floor(latencies.length * 0.5)] || 0,
1258
1338
  p95: latencies[Math.floor(latencies.length * 0.95)] || 0,
@@ -1318,32 +1398,33 @@ class LemmaServer {
1318
1398
  }
1319
1399
  });
1320
1400
  this.app.post('/api/cache/clear', async (req, res) => {
1321
- const { provider } = req.body || {};
1401
+ // resetHistory must be asked for explicitly: the ledger and stats are persisted lifetime
1402
+ // accounting, and a "clear cache" click used to wipe them to disk as a side effect.
1403
+ const { provider, resetHistory } = req.body || {};
1322
1404
  let clearedCount = 0;
1323
1405
  if (!provider) {
1324
1406
  // Global clear
1325
1407
  clearedCount = CACHE.size;
1326
1408
  CACHE.clear();
1327
1409
  EVENT_LOG.length = 0; // Clear live event log
1328
- SavingsLedger_1.savingsLedger.reset(); // Reset savings counters
1329
- // Reset and persist statistics for the current project
1330
- if (this.stats[this.projectName]) {
1331
- this.stats[this.projectName] = {
1332
- total: 0,
1333
- hits: 0,
1334
- misses: 0,
1335
- totalLatency: 0,
1336
- totalTokensSaved: 0,
1337
- providers: {}
1338
- };
1410
+ if (resetHistory === true) {
1411
+ SavingsLedger_1.savingsLedger.reset();
1412
+ }
1413
+ // Stats are per project but reported machine-wide, so a history reset covers them all.
1414
+ if (resetHistory === true) {
1415
+ for (const key of Object.keys(this.stats))
1416
+ delete this.stats[key];
1339
1417
  writeJson(ProjectStore_1.STATS_FILE, this.stats);
1340
1418
  }
1341
- // Clear Chroma DB collection
1419
+ // Completions are cached per project, so clearing the cache clears every project's
1420
+ // collection — not only the one this daemon happened to be started for.
1342
1421
  try {
1343
- await chroma.deleteCollection({ name: getCollectionName(this.projectName) }).catch(() => { });
1422
+ for (const name of await listCacheCollections()) {
1423
+ await chroma.deleteCollection({ name }).catch(() => { });
1424
+ }
1344
1425
  }
1345
1426
  catch (e) {
1346
- console.error(`[CacheClear] Failed to clear Chroma collection: ${e.message}`);
1427
+ console.error(`[CacheClear] Failed to clear Chroma collections: ${e.message}`);
1347
1428
  }
1348
1429
  }
1349
1430
  else {
@@ -1354,14 +1435,16 @@ class LemmaServer {
1354
1435
  clearedCount++;
1355
1436
  }
1356
1437
  }
1357
- // Selective delete from Chroma DB collection using provider metadata filter
1438
+ // Selective delete, by provider metadata, across every project's collection
1358
1439
  try {
1359
- const collection = await chroma.getOrCreateCollection({
1360
- name: getCollectionName(this.projectName),
1361
- embeddingFunction: dummyEmbeddingFunction,
1362
- metadata: { "hnsw:space": "cosine" }
1363
- });
1364
- await collection.delete({ where: { provider } });
1440
+ for (const name of await listCacheCollections()) {
1441
+ const collection = await chroma.getOrCreateCollection({
1442
+ name,
1443
+ embeddingFunction: dummyEmbeddingFunction,
1444
+ metadata: { "hnsw:space": "cosine" }
1445
+ });
1446
+ await collection.delete({ where: { provider } });
1447
+ }
1365
1448
  }
1366
1449
  catch (e) {
1367
1450
  console.error(`[CacheClear] Failed to selectively delete provider ${provider} from Chroma: ${e.message}`);
@@ -1386,8 +1469,40 @@ class LemmaServer {
1386
1469
  res.status(500).json({ error: e.message });
1387
1470
  }
1388
1471
  });
1472
+ // Dashboard search over Brain memories. /api/search queries the completions cache in Chroma
1473
+ // and needs Ollama for embeddings; this needs neither. countStats:false — browsing the
1474
+ // corpus from the dashboard must not count as a recall or credit reuse to any memory.
1475
+ this.app.get('/api/brain/search', async (req, res) => {
1476
+ const query = typeof req.query.q === 'string' ? req.query.q.trim() : '';
1477
+ if (!query)
1478
+ return res.status(400).json({ error: 'Query required' });
1479
+ try {
1480
+ // Same floor as search_memory, so the dashboard shows what the model would actually get.
1481
+ const results = await (0, TheBrainV2_1.getBrain)().searchHybrid(query, 10, utils_1.DEFAULT_MEMORY_SIMILARITY_FLOOR, { countStats: false });
1482
+ res.json({
1483
+ results: results.map((r) => ({
1484
+ id: r.id,
1485
+ prompt: r.query,
1486
+ response: r.response,
1487
+ timestamp: r.timestamp,
1488
+ similarity: r.similarity,
1489
+ fresh: r.fresh,
1490
+ untracked: !!r.untracked,
1491
+ })),
1492
+ });
1493
+ }
1494
+ catch (e) {
1495
+ res.status(500).json({ error: e.message });
1496
+ }
1497
+ });
1389
1498
  // Serve codebase timeline snapshots (AST Time-Travel Telemetry)
1499
+ // Snapshots come from watching this daemon's own cwd. Like onboarding, decline a caller from
1500
+ // another project instead of serving this project's history as theirs.
1390
1501
  this.app.get('/api/project/timeline', (req, res) => {
1502
+ const expectedProject = typeof req.query.expectedProject === 'string' ? req.query.expectedProject : null;
1503
+ if ((0, ProjectIdentity_1.isProjectMismatch)(expectedProject, this.projectName)) {
1504
+ return res.json({ success: false, mismatch: true, servingProject: this.projectName, timeline: [] });
1505
+ }
1391
1506
  res.json({ success: true, timeline: this.timeline });
1392
1507
  });
1393
1508
  // Dynamic Zero-Shot Onboarding mental model builder
@@ -1566,36 +1681,15 @@ class LemmaServer {
1566
1681
  tk = Math.floor(Math.max(0, charsBefore - charsAfter) / 4);
1567
1682
  SavingsLedger_1.savingsLedger.record(source, charsBefore, charsAfter);
1568
1683
  }
1569
- // Update active project stats to immediately reflect on Dashboard total requests & charts
1684
+ // A saving reported by an MCP tool is not a proxy completion: it used to bump the
1685
+ // completion stats (total++, a "miss" for every non-cache source, a mock 15ms latency),
1686
+ // which skewed the proxy's hit rate, request count and average latency. Ledger only;
1687
+ // the event just shows the saving in the activity feed, with nothing invented.
1570
1688
  if (tk > 0) {
1571
- if (!this.stats[this.projectName]) {
1572
- this.stats[this.projectName] = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0, providers: {} };
1573
- }
1574
- const s = this.stats[this.projectName];
1575
- s.total++;
1576
- s.totalTokensSaved += tk;
1577
- s.totalLatency += 15; // mock latency for local MCP metrics
1578
- if (source === 'cache') {
1579
- s.hits++;
1580
- }
1581
- else {
1582
- s.misses++;
1583
- }
1584
- if (!s.providers['mcp'])
1585
- s.providers['mcp'] = { hits: 0, misses: 0 };
1586
- if (source === 'cache')
1587
- s.providers['mcp'].hits++;
1588
- else
1589
- s.providers['mcp'].misses++;
1590
- // Write updated stats back to disk immediately
1591
- writeJson(ProjectStore_1.STATS_FILE, this.stats);
1592
- // Push to live telemetry logEvent array so the Dashboard event timeline updates instantly!
1593
1689
  logEvent({
1594
- type: source === 'cache' ? 'cache:hit' : source,
1595
- project: this.projectName,
1596
- latency: 15,
1690
+ type: source === 'cache' ? 'brain_hit' : source,
1597
1691
  provider: 'mcp',
1598
- tokens: tk
1692
+ tokens: tk,
1599
1693
  });
1600
1694
  }
1601
1695
  res.json({ ok: true, snapshot: SavingsLedger_1.savingsLedger.getSnapshot().total });
@@ -1645,11 +1739,15 @@ class LemmaServer {
1645
1739
  result: event.result || 'UNKNOWN',
1646
1740
  latency: event.latency || 0,
1647
1741
  tokensImpact: event.tokensImpact || 0,
1648
- error: event.error || null,
1742
+ error: typeof event.error === 'string' ? event.error.slice(0, 200) : null,
1743
+ receipt: typeof event.receipt === 'string' ? event.receipt : undefined,
1744
+ projectPath: typeof event.projectPath === 'string' ? event.projectPath : undefined,
1649
1745
  timestamp: event.timestamp || new Date().toISOString(),
1650
1746
  };
1651
1747
  // Add to EVENT_LOG
1652
1748
  logEvent(mcpEvent);
1749
+ if (mcpEvent.projectPath)
1750
+ this.registry.recordToolCall(mcpEvent.projectPath);
1653
1751
  res.json({ success: true, eventId: mcpEvent.id });
1654
1752
  }
1655
1753
  catch (error) {
@@ -1850,6 +1948,12 @@ class LemmaServer {
1850
1948
  const tier = pro ? 'pro' : 'standard';
1851
1949
  const isStream = !!req.body.stream;
1852
1950
  let historyTokensSaved = 0;
1951
+ // The daemon is one process for every project on the machine; its own projectName is just
1952
+ // whichever cwd it was started from. Attribute each completion — stats, and above all the
1953
+ // semantic-cache collection — to the project the request comes from (MCP handshake
1954
+ // registry, then the messages themselves), so one project's cached answer is never served
1955
+ // to another. Falls back to the daemon's project when nothing identifies the caller.
1956
+ const requestProject = this.detectActiveProject(req)?.name ?? this.projectName;
1853
1957
  if ((0, DebuggerState_1.isDebuggerPaused)()) {
1854
1958
  res.setHeader('Retry-After', '5');
1855
1959
  return res.status(503).json({ error: { message: 'Lemma is paused from the dashboard debugger. Resume it to continue.' } });
@@ -1865,15 +1969,17 @@ class LemmaServer {
1865
1969
  return res.status(400).json({ error: 'Invalid request format' });
1866
1970
  const { maskedPrompt: prompt, tokenMap } = semanticScrubber.mask(rawPrompt);
1867
1971
  if (Object.keys(tokenMap).length > 0) {
1868
- logEvent({ type: 'privacy:mask', count: Object.keys(tokenMap).length, project: this.projectName });
1972
+ logEvent({ type: 'privacy:mask', count: Object.keys(tokenMap).length, project: requestProject });
1973
+ SECRETS_MASKED.count += Object.keys(tokenMap).length;
1869
1974
  }
1870
1975
  const originalModel = req.body.model || 'gpt-4o';
1871
1976
  // ── 1. Cache-First Hot Path ──
1872
1977
  // Perform exact cache check immediately (resolves in <1ms)
1873
1978
  const hit = cacheGet(provider, prompt);
1874
1979
  if (hit) {
1875
- const cacheTokensSaved = 2000;
1876
- await recordStat(this.stats, this.projectName, true, Date.now() - t0, provider, cacheTokensSaved);
1980
+ // Measured from the cached response, like the semantic path — a flat 2000 used to go
1981
+ // straight into the ledger as if it had been counted.
1982
+ await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(hit.data));
1877
1983
  const unmaskedData = semanticScrubber.unmask(hit.data, tokenMap);
1878
1984
  if (isStream)
1879
1985
  return SseRelay_1.SseRelay.simulateHit(res, unmaskedData, { provider: provider, tier, similarity: 1.0 });
@@ -1884,10 +1990,10 @@ class LemmaServer {
1884
1990
  }
1885
1991
  // Perform semantic cache check (resolves in ~15ms)
1886
1992
  if (pro) {
1887
- const semHit = await semanticGet(provider, prompt, this.projectName);
1993
+ const semHit = await semanticGet(provider, prompt, requestProject);
1888
1994
  if (semHit) {
1889
1995
  if (semHit.similarity >= 0.90) {
1890
- await recordStat(this.stats, this.projectName, true, Date.now() - t0, provider, measureCachedTokens(semHit.data));
1996
+ await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(semHit.data));
1891
1997
  const unmaskedData = semanticScrubber.unmask(semHit.data, tokenMap);
1892
1998
  const cacheHeader = semHit.hiveMind ? 'HIT-HIVE-MIND' : 'HIT';
1893
1999
  if (isStream)
@@ -1938,7 +2044,7 @@ Adjusted Answer:`;
1938
2044
  else if (provider === 'anthropic') {
1939
2045
  synthesizedData.content[0].text = adjustedText;
1940
2046
  }
1941
- await recordStat(this.stats, this.projectName, true, Date.now() - t0, provider, measureCachedTokens(synthesizedData));
2047
+ await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(synthesizedData));
1942
2048
  const unmaskedData = semanticScrubber.unmask(synthesizedData, tokenMap);
1943
2049
  if (isStream) {
1944
2050
  return SseRelay_1.SseRelay.simulateHit(res, unmaskedData, { provider: provider, tier, similarity: semHit.similarity });
@@ -1958,7 +2064,7 @@ Adjusted Answer:`;
1958
2064
  const cloudHit = await cloudSync.get(prompt);
1959
2065
  if (cloudHit) {
1960
2066
  cacheSet(provider, prompt, cloudHit.data);
1961
- await recordStat(this.stats, this.projectName, true, Date.now() - t0, provider, measureCachedTokens(cloudHit.data));
2067
+ await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(cloudHit.data));
1962
2068
  const unmaskedData = semanticScrubber.unmask(cloudHit.data, tokenMap);
1963
2069
  if (isStream)
1964
2070
  return SseRelay_1.SseRelay.simulateHit(res, unmaskedData, { provider: provider, tier, similarity: cloudHit.similarity || 0.95 });
@@ -2054,7 +2160,7 @@ Adjusted Answer:`;
2054
2160
  }
2055
2161
  }
2056
2162
  if (isStream) {
2057
- await recordStat(this.stats, this.projectName, false, Date.now() - t0, provider, historyTokensSaved);
2163
+ await recordStat(this.stats, requestProject, false, Date.now() - t0, provider, historyTokensSaved);
2058
2164
  if (!pro) {
2059
2165
  await usageWriteQueue.enqueue(async () => {
2060
2166
  const u = await loadUsage();
@@ -2068,7 +2174,7 @@ Adjusted Answer:`;
2068
2174
  const data = await this.callUpstream(req.body, provider);
2069
2175
  cacheSet(provider, prompt, data);
2070
2176
  if (pro) {
2071
- await semanticSet(provider, prompt, data, this.projectName);
2177
+ await semanticSet(provider, prompt, data, requestProject);
2072
2178
  await cloudSync.set(prompt, data);
2073
2179
  }
2074
2180
  if (!pro) {
@@ -2078,7 +2184,7 @@ Adjusted Answer:`;
2078
2184
  await writeJson(ProjectStore_1.USAGE_FILE, u);
2079
2185
  });
2080
2186
  }
2081
- await recordStat(this.stats, this.projectName, false, Date.now() - t0, provider, historyTokensSaved);
2187
+ await recordStat(this.stats, requestProject, false, Date.now() - t0, provider, historyTokensSaved);
2082
2188
  return res.json(semanticScrubber.unmask(data, tokenMap));
2083
2189
  }
2084
2190
  catch (err) {
@@ -2230,23 +2336,31 @@ Adjusted Answer:`;
2230
2336
  console.log('simply copy-paste this command into your MCP settings:');
2231
2337
  console.log(`\n \x1b[1m\x1b[33mlemma mcp\x1b[0m\n`);
2232
2338
  console.log('────────────────────────────────────────────────\n');
2233
- // Auto-start the Local Clipboard Optimizer Watcher in the background
2234
- const clipboardDisabled = process.env.LEMMA_DISABLE_CLIPBOARD === 'true';
2235
- if (clipboardDisabled) {
2339
+ // The clipboard watcher runs as its own background process (startBackgroundClipboardWatcher,
2340
+ // tracked in CLIPBOARD_PID_FILE and stopped by `lemma stop`). This used to also start a
2341
+ // second, in-process watcher here: two pollers on one clipboard squeezed and reported the
2342
+ // same copy twice.
2343
+ if (process.env.LEMMA_DISABLE_CLIPBOARD === 'true') {
2236
2344
  console.log(`📋 Clipboard Watcher : \x1b[90m[DISABLED via LEMMA_DISABLE_CLIPBOARD]\x1b[0m`);
2237
2345
  }
2238
- else {
2239
- console.log(`📋 Clipboard Watcher : \x1b[33m[ACTIVE]\x1b[0m — polling clipboard every 1s (disable: LEMMA_DISABLE_CLIPBOARD=true)`);
2240
- try {
2241
- const { startClipboardWatcher } = require('../utils/ClipboardWatcher');
2242
- startClipboardWatcher(1000);
2243
- }
2244
- catch (err) {
2245
- console.warn(`⚠️ Could not auto-initialize Clipboard Watcher: ${err.message}`);
2246
- }
2247
- }
2248
2346
  });
2249
- process.on('SIGTERM', () => server.close());
2347
+ // A SIGTERM listener replaces Node's default exit, and server.close() only stops accepting:
2348
+ // keep-alive and socket.io connections, file watchers and intervals keep the loop alive, so
2349
+ // `lemma stop` used to leave a daemon with no port that never exited. Flush the Brain's
2350
+ // debounced write (same reason as the MCP bootstrap), close, and exit — with a hard cap.
2351
+ const shutdown = () => {
2352
+ try {
2353
+ (0, TheBrainV2_1.getBrain)().flushSync();
2354
+ }
2355
+ catch {
2356
+ // A flush failure must never block shutdown — the next save() heals.
2357
+ }
2358
+ server.close(() => process.exit(0));
2359
+ server.closeAllConnections();
2360
+ setTimeout(() => process.exit(0), 3000).unref();
2361
+ };
2362
+ process.once('SIGTERM', shutdown);
2363
+ process.once('SIGINT', shutdown);
2250
2364
  }
2251
2365
  }
2252
2366
  // ── CLI ────────────────────────────────────────────────────────────────────────
@@ -2941,6 +3055,20 @@ commander_1.program.command('clipboard')
2941
3055
  .option('--interval <ms>', 'Polling interval in milliseconds', '1000')
2942
3056
  .action((opts) => {
2943
3057
  try {
3058
+ // One watcher per machine: every poller on the same clipboard squeezes and reports the
3059
+ // same copy again. Defer to a live watcher recorded in the pid file; otherwise record us.
3060
+ try {
3061
+ const recorded = parseInt(fs_1.default.readFileSync(ProjectStore_1.CLIPBOARD_PID_FILE, 'utf8'), 10);
3062
+ if (recorded && recorded !== process.pid) {
3063
+ process.kill(recorded, 0); // throws if it's gone
3064
+ console.log(`✅ [Clipboard] Watcher already running (PID: ${recorded}) — not starting another.`);
3065
+ process.exit(0);
3066
+ }
3067
+ }
3068
+ catch {
3069
+ // No pid file, or its process is dead: this one is the watcher.
3070
+ }
3071
+ fs_1.default.writeFileSync(ProjectStore_1.CLIPBOARD_PID_FILE, String(process.pid));
2944
3072
  const interval = parseInt(opts.interval) || 1000;
2945
3073
  const { startClipboardWatcher } = require('../utils/ClipboardWatcher');
2946
3074
  startClipboardWatcher(interval);