@nxuss/lemma 1.35.0 → 1.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -426
- package/dashboard/README.md +48 -19
- package/dashboard/dist/assets/index-BoeQau6n.css +1 -0
- package/dashboard/dist/assets/index-Clf3qR64.js +451 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/cjs/api/dashboardRoutes.d.ts.map +1 -1
- package/dist/cjs/api/dashboardRoutes.js +7 -3
- package/dist/cjs/api/dashboardRoutes.js.map +1 -1
- package/dist/cjs/autopilot/Notifier.d.ts.map +1 -1
- package/dist/cjs/autopilot/Notifier.js +3 -0
- package/dist/cjs/autopilot/Notifier.js.map +1 -1
- package/dist/cjs/cli/lemma-proxy.d.ts.map +1 -1
- package/dist/cjs/cli/lemma-proxy.js +214 -103
- package/dist/cjs/cli/lemma-proxy.js.map +1 -1
- package/dist/cjs/mcp/index.js +9 -8
- package/dist/cjs/mcp/index.js.map +1 -1
- package/dist/cjs/mcp/resources.d.ts.map +1 -1
- package/dist/cjs/mcp/resources.js +23 -4
- package/dist/cjs/mcp/resources.js.map +1 -1
- package/dist/cjs/mcp/tools/memory.js +11 -11
- package/dist/cjs/mcp/tools/memory.js.map +1 -1
- package/dist/cjs/mcp/tools/misc.js +1 -1
- package/dist/cjs/mcp/tools/misc.js.map +1 -1
- package/dist/cjs/mcp/tools/review.js +1 -1
- package/dist/cjs/mcp/tools/review.js.map +1 -1
- package/dist/cjs/mcp/tools/workspace.d.ts.map +1 -1
- package/dist/cjs/mcp/tools/workspace.js +10 -7
- package/dist/cjs/mcp/tools/workspace.js.map +1 -1
- package/dist/cjs/mcp/tools.d.ts +16 -1
- package/dist/cjs/mcp/tools.d.ts.map +1 -1
- package/dist/cjs/mcp/tools.js +63 -33
- package/dist/cjs/mcp/tools.js.map +1 -1
- package/dist/cjs/utils/ProjectIdentity.d.ts +8 -0
- package/dist/cjs/utils/ProjectIdentity.d.ts.map +1 -1
- package/dist/cjs/utils/ProjectIdentity.js +15 -0
- package/dist/cjs/utils/ProjectIdentity.js.map +1 -1
- package/dist/cjs/utils/SavingsLedger.d.ts +2 -0
- package/dist/cjs/utils/SavingsLedger.d.ts.map +1 -1
- package/dist/cjs/utils/SavingsLedger.js +5 -5
- package/dist/cjs/utils/SavingsLedger.js.map +1 -1
- package/dist/cjs/utils/TokenReceipt.d.ts +2 -0
- package/dist/cjs/utils/TokenReceipt.d.ts.map +1 -1
- package/dist/cjs/utils/TokenReceipt.js +11 -1
- package/dist/cjs/utils/TokenReceipt.js.map +1 -1
- package/dist/cjs/utils/reportSavings.d.ts +7 -0
- package/dist/cjs/utils/reportSavings.d.ts.map +1 -1
- package/dist/cjs/utils/reportSavings.js +14 -2
- package/dist/cjs/utils/reportSavings.js.map +1 -1
- package/dist/esm/api/dashboardRoutes.d.ts.map +1 -1
- package/dist/esm/api/dashboardRoutes.js +7 -3
- package/dist/esm/api/dashboardRoutes.js.map +1 -1
- package/dist/esm/autopilot/Notifier.d.ts.map +1 -1
- package/dist/esm/autopilot/Notifier.js +3 -0
- package/dist/esm/autopilot/Notifier.js.map +1 -1
- package/dist/esm/cli/lemma-proxy.d.ts.map +1 -1
- package/dist/esm/cli/lemma-proxy.js +215 -104
- package/dist/esm/cli/lemma-proxy.js.map +1 -1
- package/dist/esm/mcp/index.js +9 -8
- package/dist/esm/mcp/index.js.map +1 -1
- package/dist/esm/mcp/resources.d.ts.map +1 -1
- package/dist/esm/mcp/resources.js +23 -4
- package/dist/esm/mcp/resources.js.map +1 -1
- package/dist/esm/mcp/tools/memory.js +11 -11
- package/dist/esm/mcp/tools/memory.js.map +1 -1
- package/dist/esm/mcp/tools/misc.js +1 -1
- package/dist/esm/mcp/tools/misc.js.map +1 -1
- package/dist/esm/mcp/tools/review.js +1 -1
- package/dist/esm/mcp/tools/review.js.map +1 -1
- package/dist/esm/mcp/tools/workspace.d.ts.map +1 -1
- package/dist/esm/mcp/tools/workspace.js +10 -7
- package/dist/esm/mcp/tools/workspace.js.map +1 -1
- package/dist/esm/mcp/tools.d.ts +16 -1
- package/dist/esm/mcp/tools.d.ts.map +1 -1
- package/dist/esm/mcp/tools.js +62 -34
- package/dist/esm/mcp/tools.js.map +1 -1
- package/dist/esm/utils/ProjectIdentity.d.ts +8 -0
- package/dist/esm/utils/ProjectIdentity.d.ts.map +1 -1
- package/dist/esm/utils/ProjectIdentity.js +14 -0
- package/dist/esm/utils/ProjectIdentity.js.map +1 -1
- package/dist/esm/utils/SavingsLedger.d.ts +2 -0
- package/dist/esm/utils/SavingsLedger.d.ts.map +1 -1
- package/dist/esm/utils/SavingsLedger.js +1 -1
- package/dist/esm/utils/SavingsLedger.js.map +1 -1
- package/dist/esm/utils/TokenReceipt.d.ts +2 -0
- package/dist/esm/utils/TokenReceipt.d.ts.map +1 -1
- package/dist/esm/utils/TokenReceipt.js +10 -1
- package/dist/esm/utils/TokenReceipt.js.map +1 -1
- package/dist/esm/utils/reportSavings.d.ts +7 -0
- package/dist/esm/utils/reportSavings.d.ts.map +1 -1
- package/dist/esm/utils/reportSavings.js +13 -2
- package/dist/esm/utils/reportSavings.js.map +1 -1
- package/package.json +10 -3
- package/dashboard/dist/assets/index-Yt1H2B7F.css +0 -1
- package/dashboard/dist/assets/index-euG1IvQ7.js +0 -481
|
@@ -61,6 +61,8 @@ const ProjectStore_1 = require("../proxy/ProjectStore");
|
|
|
61
61
|
const gitHooks_1 = require("./gitHooks");
|
|
62
62
|
const hostHooks_1 = require("./hostHooks");
|
|
63
63
|
const TheBrainV2_1 = require("../subconscious/TheBrainV2");
|
|
64
|
+
const utils_1 = require("../mcp/utils");
|
|
65
|
+
const ProjectIdentity_2 = require("../utils/ProjectIdentity");
|
|
64
66
|
const chroma = new chromadb_1.ChromaClient({ host: 'localhost', port: 8000 });
|
|
65
67
|
const dummyEmbeddingFunction = { generate: (_texts) => Promise.resolve([]) };
|
|
66
68
|
const complexityRouter = new ComplexityRouter_1.default();
|
|
@@ -96,7 +98,7 @@ function writeJson(filePath, data) {
|
|
|
96
98
|
catch { }
|
|
97
99
|
}
|
|
98
100
|
function getCollectionName(projectName) {
|
|
99
|
-
return
|
|
101
|
+
return (0, ProjectIdentity_2.cacheCollectionName)(projectName);
|
|
100
102
|
}
|
|
101
103
|
// detectProject/isProjectMismatch live in ../utils/ProjectIdentity — a side-effect-free
|
|
102
104
|
// module shared with src/mcp/tools.ts, so the client and this daemon can never define
|
|
@@ -396,6 +398,11 @@ enabled = true
|
|
|
396
398
|
}
|
|
397
399
|
const EVENT_LOG = [];
|
|
398
400
|
const MAX_EVENTS = 100;
|
|
401
|
+
/**
|
|
402
|
+
* Secrets masked since this daemon started. Counting 'privacy:mask' entries in EVENT_LOG only
|
|
403
|
+
* ever saw the last 100 events, and counted requests rather than secrets.
|
|
404
|
+
*/
|
|
405
|
+
const SECRETS_MASKED = { count: 0 };
|
|
399
406
|
// MCP Status tracking
|
|
400
407
|
const MCP_STATUS = {
|
|
401
408
|
lastActivity: Date.now(),
|
|
@@ -482,7 +489,7 @@ function startBackgroundClipboardWatcher(cliOpts) {
|
|
|
482
489
|
}
|
|
483
490
|
}
|
|
484
491
|
catch { }
|
|
485
|
-
const shouldWatch = cliOpts?.clipboard !== false && !configDisabled;
|
|
492
|
+
const shouldWatch = cliOpts?.clipboard !== false && !configDisabled && process.env.LEMMA_DISABLE_CLIPBOARD !== 'true';
|
|
486
493
|
if (!shouldWatch) {
|
|
487
494
|
console.log(`⏭️ [Clipboard] Background clipboard watcher is disabled (via options or config).`);
|
|
488
495
|
return;
|
|
@@ -590,6 +597,20 @@ async function loadStats() {
|
|
|
590
597
|
* Tokens a cache hit actually avoided regenerating, measured from the cached payload
|
|
591
598
|
* that was served instead of an LLM call. Replaces the old hardcoded 2000 guess.
|
|
592
599
|
*/
|
|
600
|
+
/** The answer text inside a cached provider response (OpenAI, Anthropic or Gemini shape). */
|
|
601
|
+
function extractResponseText(data) {
|
|
602
|
+
const text = data?.choices?.[0]?.message?.content ??
|
|
603
|
+
data?.content?.find?.((c) => c?.type === 'text')?.text ??
|
|
604
|
+
data?.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
605
|
+
if (typeof text === 'string')
|
|
606
|
+
return text;
|
|
607
|
+
try {
|
|
608
|
+
return JSON.stringify(data);
|
|
609
|
+
}
|
|
610
|
+
catch {
|
|
611
|
+
return '';
|
|
612
|
+
}
|
|
613
|
+
}
|
|
593
614
|
function measureCachedTokens(data) {
|
|
594
615
|
try {
|
|
595
616
|
const chars = JSON.stringify(data)?.length;
|
|
@@ -611,6 +632,33 @@ function measureCachedTokens(data) {
|
|
|
611
632
|
* round-trip), so a miss costs latency, not tokens. Charging it a fabricated token
|
|
612
633
|
* figure would repeat exactly the error the cost ledger exists to correct.
|
|
613
634
|
*/
|
|
635
|
+
/**
|
|
636
|
+
* Machine-wide totals. Completions are attributed per project (see handleCompletion), and the
|
|
637
|
+
* dashboard — like the savings ledger — reports the whole daemon, not one project's slice.
|
|
638
|
+
*/
|
|
639
|
+
function aggregateStats(stats) {
|
|
640
|
+
const sum = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0 };
|
|
641
|
+
for (const s of Object.values(stats || {})) {
|
|
642
|
+
// Stats written before /api/savings-record stopped touching them counted every MCP saving
|
|
643
|
+
// as a completion under provider 'mcp', with a mock 15ms latency. Those counts are exact in
|
|
644
|
+
// the provider breakdown, so take them back out; only real completions remain.
|
|
645
|
+
const mcp = s?.providers?.mcp || { hits: 0, misses: 0 };
|
|
646
|
+
const mcpCalls = (mcp.hits || 0) + (mcp.misses || 0);
|
|
647
|
+
sum.total += Math.max(0, (s?.total || 0) - mcpCalls);
|
|
648
|
+
sum.hits += Math.max(0, (s?.hits || 0) - (mcp.hits || 0));
|
|
649
|
+
sum.misses += Math.max(0, (s?.misses || 0) - (mcp.misses || 0));
|
|
650
|
+
sum.totalLatency += Math.max(0, (s?.totalLatency || 0) - mcpCalls * 15);
|
|
651
|
+
sum.totalTokensSaved += s?.totalTokensSaved || 0;
|
|
652
|
+
}
|
|
653
|
+
return sum;
|
|
654
|
+
}
|
|
655
|
+
/** Every semantic-cache collection the daemon writes: one per project, all "lemma-cache" prefixed. */
|
|
656
|
+
async function listCacheCollections() {
|
|
657
|
+
const cols = await chroma.listCollections();
|
|
658
|
+
return cols
|
|
659
|
+
.map((c) => (typeof c === 'string' ? c : c?.name))
|
|
660
|
+
.filter((n) => typeof n === 'string' && n.startsWith('lemma-cache'));
|
|
661
|
+
}
|
|
614
662
|
async function recordStat(stats, project, fromCache, latencyMs, provider, tokensSaved) {
|
|
615
663
|
if (!stats[project]) {
|
|
616
664
|
stats[project] = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0, providers: {} };
|
|
@@ -622,7 +670,8 @@ async function recordStat(stats, project, fromCache, latencyMs, provider, tokens
|
|
|
622
670
|
s.hits++;
|
|
623
671
|
else
|
|
624
672
|
s.misses++;
|
|
625
|
-
|
|
673
|
+
// Unmeasured is 0, not a guess: a 2000 placeholder used to surface as "2000 tokens saved".
|
|
674
|
+
const tkSaved = tokensSaved || 0;
|
|
626
675
|
s.totalTokensSaved += tkSaved;
|
|
627
676
|
if (!s.providers[provider])
|
|
628
677
|
s.providers[provider] = { hits: 0, misses: 0 };
|
|
@@ -630,9 +679,7 @@ async function recordStat(stats, project, fromCache, latencyMs, provider, tokens
|
|
|
630
679
|
s.providers[provider].hits++;
|
|
631
680
|
else
|
|
632
681
|
s.providers[provider].misses++;
|
|
633
|
-
// Only measured savings reach the ledger.
|
|
634
|
-
// caller never counted, so tkSaved is the 2000 placeholder — usable for the
|
|
635
|
-
// dashboard, not for a net-position claim.
|
|
682
|
+
// Only measured savings reach the ledger.
|
|
636
683
|
if (fromCache && typeof tokensSaved === 'number' && tokensSaved > 0) {
|
|
637
684
|
SavingsLedger_1.savingsLedger.recordTokens('cache', tokensSaved);
|
|
638
685
|
}
|
|
@@ -659,6 +706,9 @@ function cacheGet(provider, prompt) {
|
|
|
659
706
|
CACHE.delete(k);
|
|
660
707
|
return null;
|
|
661
708
|
}
|
|
709
|
+
// The only caller is the exact-hit path, so a returned entry is a served hit.
|
|
710
|
+
e.hits++;
|
|
711
|
+
e.lastHitAt = Date.now();
|
|
662
712
|
return e;
|
|
663
713
|
}
|
|
664
714
|
function cacheSet(provider, prompt, data) {
|
|
@@ -670,7 +720,7 @@ function cacheSet(provider, prompt, data) {
|
|
|
670
720
|
if (oldestKey)
|
|
671
721
|
CACHE.delete(oldestKey);
|
|
672
722
|
}
|
|
673
|
-
CACHE.set(k, { input: prompt, data, createdAt: Date.now(), ttl: TTL_MS, provider });
|
|
723
|
+
CACHE.set(k, { input: prompt, data, createdAt: Date.now(), ttl: TTL_MS, provider, hits: 0 });
|
|
674
724
|
}
|
|
675
725
|
// ── Semantic Search (Pro) ──────────────────────────────────────────────────────
|
|
676
726
|
async function getEmbedding(text) {
|
|
@@ -1096,10 +1146,10 @@ class LemmaServer {
|
|
|
1096
1146
|
if (fs_1.default.existsSync(this.dashboardPath)) {
|
|
1097
1147
|
this.app.use('/dashboard/', express_1.default.static(this.dashboardPath));
|
|
1098
1148
|
this.app.get('/api/metrics', async (req, res) => {
|
|
1099
|
-
const stats = this.stats
|
|
1149
|
+
const stats = aggregateStats(this.stats);
|
|
1100
1150
|
const hitRate = stats.total > 0 ? stats.hits / stats.total : 0;
|
|
1101
1151
|
const avgLat = stats.total > 0 ? stats.totalLatency / stats.total : 0;
|
|
1102
|
-
const secretsMasked =
|
|
1152
|
+
const secretsMasked = SECRETS_MASKED.count;
|
|
1103
1153
|
const ledgerSnap = SavingsLedger_1.savingsLedger.getSnapshot();
|
|
1104
1154
|
let activeAgents = 1; // Always include the Gateway itself
|
|
1105
1155
|
let totalAgents = 1;
|
|
@@ -1130,8 +1180,7 @@ class LemmaServer {
|
|
|
1130
1180
|
});
|
|
1131
1181
|
this.app.get('/api/agents', async (req, res) => {
|
|
1132
1182
|
// Load statistics for the Gateway itself
|
|
1133
|
-
const
|
|
1134
|
-
const statsProj = stats[this.projectName] || { total: 0, hits: 0 };
|
|
1183
|
+
const statsProj = aggregateStats(await loadStats());
|
|
1135
1184
|
const total = statsProj.total || 0;
|
|
1136
1185
|
const hits = statsProj.hits || 0;
|
|
1137
1186
|
const gatewayAgent = {
|
|
@@ -1168,7 +1217,8 @@ class LemmaServer {
|
|
|
1168
1217
|
type: e.type, // e.g., 'cache:hit' or 'cache:miss' which client normalizes to 'cache_hit' or 'cache_miss'
|
|
1169
1218
|
latency: e.latency,
|
|
1170
1219
|
tokensSaved: e.tokens || e.tokensImpact || 0,
|
|
1171
|
-
|
|
1220
|
+
// Same per-token price as the savings ledger, so a row and the totals agree.
|
|
1221
|
+
costSaved: (e.tokens || e.tokensImpact || 0) * SavingsLedger_1.AVG_COST_PER_TOKEN,
|
|
1172
1222
|
metadata: {
|
|
1173
1223
|
provider: e.provider,
|
|
1174
1224
|
project: e.project
|
|
@@ -1180,6 +1230,12 @@ class LemmaServer {
|
|
|
1180
1230
|
eventData.args = e.args;
|
|
1181
1231
|
eventData.result = e.result;
|
|
1182
1232
|
eventData.tokensImpact = e.tokensImpact;
|
|
1233
|
+
eventData.error = e.error;
|
|
1234
|
+
eventData.receipt = e.receipt;
|
|
1235
|
+
eventData.projectPath = e.projectPath;
|
|
1236
|
+
// tokensImpact is the size of the tool result — a context cost, not a saving.
|
|
1237
|
+
eventData.tokensSaved = 0;
|
|
1238
|
+
eventData.costSaved = 0;
|
|
1183
1239
|
}
|
|
1184
1240
|
return eventData;
|
|
1185
1241
|
});
|
|
@@ -1213,28 +1269,34 @@ class LemmaServer {
|
|
|
1213
1269
|
this.app.get('/api/cache-stats', async (req, res) => {
|
|
1214
1270
|
let topQueries = [];
|
|
1215
1271
|
let totalCached = CACHE.size;
|
|
1216
|
-
//
|
|
1272
|
+
// Semantic entries across every project's collection (completions are cached per project)
|
|
1217
1273
|
try {
|
|
1218
|
-
const
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1274
|
+
for (const name of await listCacheCollections()) {
|
|
1275
|
+
const collection = await chroma.getOrCreateCollection({
|
|
1276
|
+
name,
|
|
1277
|
+
embeddingFunction: dummyEmbeddingFunction,
|
|
1278
|
+
metadata: { "hnsw:space": "cosine" }
|
|
1279
|
+
});
|
|
1280
|
+
totalCached += await collection.count();
|
|
1281
|
+
}
|
|
1225
1282
|
}
|
|
1226
1283
|
catch { }
|
|
1227
|
-
//
|
|
1284
|
+
// Most reused answers: per-entry hit counts and the measured size of the cached response.
|
|
1285
|
+
// These used to be a flat 2000 tokens at $2/M with the provider's hit count stamped on
|
|
1286
|
+
// every entry, and "last hit" was really the creation time.
|
|
1228
1287
|
for (const [key, entry] of CACHE.entries()) {
|
|
1229
|
-
|
|
1230
|
-
|
|
1288
|
+
if (entry.hits === 0)
|
|
1289
|
+
continue;
|
|
1290
|
+
const tokensPerHit = measureCachedTokens(entry.data) || 0;
|
|
1291
|
+
const totalTokens = tokensPerHit * entry.hits;
|
|
1231
1292
|
topQueries.push({
|
|
1232
1293
|
queryHash: key.substring(0, 16),
|
|
1233
1294
|
query: entry.input,
|
|
1234
|
-
hitCount,
|
|
1235
|
-
totalTokens
|
|
1236
|
-
costSaved:
|
|
1237
|
-
lastHit: new Date(entry.createdAt).toISOString()
|
|
1295
|
+
hitCount: entry.hits,
|
|
1296
|
+
totalTokens,
|
|
1297
|
+
costSaved: totalTokens * SavingsLedger_1.AVG_COST_PER_TOKEN,
|
|
1298
|
+
lastHit: new Date(entry.lastHitAt || entry.createdAt).toISOString(),
|
|
1299
|
+
responsePreview: extractResponseText(entry.data).slice(0, 600),
|
|
1238
1300
|
});
|
|
1239
1301
|
}
|
|
1240
1302
|
// Try to query the stack orchestration API if running to append stack cache stats
|
|
@@ -1251,8 +1313,11 @@ class LemmaServer {
|
|
|
1251
1313
|
catch { }
|
|
1252
1314
|
// Sort by hitCount desc
|
|
1253
1315
|
topQueries.sort((a, b) => b.hitCount - a.hitCount);
|
|
1254
|
-
// Latency percentiles
|
|
1255
|
-
const latencies = EVENT_LOG
|
|
1316
|
+
// Latency percentiles of proxy completions only — MCP tool calls share EVENT_LOG.
|
|
1317
|
+
const latencies = EVENT_LOG
|
|
1318
|
+
.filter(e => (e.type === 'cache:hit' || e.type === 'cache:miss') && e.provider !== 'mcp' && typeof e.latency === 'number')
|
|
1319
|
+
.map(e => e.latency)
|
|
1320
|
+
.sort((a, b) => a - b);
|
|
1256
1321
|
const latencyPercentiles = {
|
|
1257
1322
|
p50: latencies[Math.floor(latencies.length * 0.5)] || 0,
|
|
1258
1323
|
p95: latencies[Math.floor(latencies.length * 0.95)] || 0,
|
|
@@ -1318,32 +1383,33 @@ class LemmaServer {
|
|
|
1318
1383
|
}
|
|
1319
1384
|
});
|
|
1320
1385
|
this.app.post('/api/cache/clear', async (req, res) => {
|
|
1321
|
-
|
|
1386
|
+
// resetHistory must be asked for explicitly: the ledger and stats are persisted lifetime
|
|
1387
|
+
// accounting, and a "clear cache" click used to wipe them to disk as a side effect.
|
|
1388
|
+
const { provider, resetHistory } = req.body || {};
|
|
1322
1389
|
let clearedCount = 0;
|
|
1323
1390
|
if (!provider) {
|
|
1324
1391
|
// Global clear
|
|
1325
1392
|
clearedCount = CACHE.size;
|
|
1326
1393
|
CACHE.clear();
|
|
1327
1394
|
EVENT_LOG.length = 0; // Clear live event log
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
totalLatency: 0,
|
|
1336
|
-
totalTokensSaved: 0,
|
|
1337
|
-
providers: {}
|
|
1338
|
-
};
|
|
1395
|
+
if (resetHistory === true) {
|
|
1396
|
+
SavingsLedger_1.savingsLedger.reset();
|
|
1397
|
+
}
|
|
1398
|
+
// Stats are per project but reported machine-wide, so a history reset covers them all.
|
|
1399
|
+
if (resetHistory === true) {
|
|
1400
|
+
for (const key of Object.keys(this.stats))
|
|
1401
|
+
delete this.stats[key];
|
|
1339
1402
|
writeJson(ProjectStore_1.STATS_FILE, this.stats);
|
|
1340
1403
|
}
|
|
1341
|
-
//
|
|
1404
|
+
// Completions are cached per project, so clearing the cache clears every project's
|
|
1405
|
+
// collection — not only the one this daemon happened to be started for.
|
|
1342
1406
|
try {
|
|
1343
|
-
|
|
1407
|
+
for (const name of await listCacheCollections()) {
|
|
1408
|
+
await chroma.deleteCollection({ name }).catch(() => { });
|
|
1409
|
+
}
|
|
1344
1410
|
}
|
|
1345
1411
|
catch (e) {
|
|
1346
|
-
console.error(`[CacheClear] Failed to clear Chroma
|
|
1412
|
+
console.error(`[CacheClear] Failed to clear Chroma collections: ${e.message}`);
|
|
1347
1413
|
}
|
|
1348
1414
|
}
|
|
1349
1415
|
else {
|
|
@@ -1354,14 +1420,16 @@ class LemmaServer {
|
|
|
1354
1420
|
clearedCount++;
|
|
1355
1421
|
}
|
|
1356
1422
|
}
|
|
1357
|
-
// Selective delete
|
|
1423
|
+
// Selective delete, by provider metadata, across every project's collection
|
|
1358
1424
|
try {
|
|
1359
|
-
const
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1425
|
+
for (const name of await listCacheCollections()) {
|
|
1426
|
+
const collection = await chroma.getOrCreateCollection({
|
|
1427
|
+
name,
|
|
1428
|
+
embeddingFunction: dummyEmbeddingFunction,
|
|
1429
|
+
metadata: { "hnsw:space": "cosine" }
|
|
1430
|
+
});
|
|
1431
|
+
await collection.delete({ where: { provider } });
|
|
1432
|
+
}
|
|
1365
1433
|
}
|
|
1366
1434
|
catch (e) {
|
|
1367
1435
|
console.error(`[CacheClear] Failed to selectively delete provider ${provider} from Chroma: ${e.message}`);
|
|
@@ -1386,8 +1454,40 @@ class LemmaServer {
|
|
|
1386
1454
|
res.status(500).json({ error: e.message });
|
|
1387
1455
|
}
|
|
1388
1456
|
});
|
|
1457
|
+
// Dashboard search over Brain memories. /api/search queries the completions cache in Chroma
|
|
1458
|
+
// and needs Ollama for embeddings; this needs neither. countStats:false — browsing the
|
|
1459
|
+
// corpus from the dashboard must not count as a recall or credit reuse to any memory.
|
|
1460
|
+
this.app.get('/api/brain/search', async (req, res) => {
|
|
1461
|
+
const query = typeof req.query.q === 'string' ? req.query.q.trim() : '';
|
|
1462
|
+
if (!query)
|
|
1463
|
+
return res.status(400).json({ error: 'Query required' });
|
|
1464
|
+
try {
|
|
1465
|
+
// Same floor as search_memory, so the dashboard shows what the model would actually get.
|
|
1466
|
+
const results = await (0, TheBrainV2_1.getBrain)().searchHybrid(query, 10, utils_1.DEFAULT_MEMORY_SIMILARITY_FLOOR, { countStats: false });
|
|
1467
|
+
res.json({
|
|
1468
|
+
results: results.map((r) => ({
|
|
1469
|
+
id: r.id,
|
|
1470
|
+
prompt: r.query,
|
|
1471
|
+
response: r.response,
|
|
1472
|
+
timestamp: r.timestamp,
|
|
1473
|
+
similarity: r.similarity,
|
|
1474
|
+
fresh: r.fresh,
|
|
1475
|
+
untracked: !!r.untracked,
|
|
1476
|
+
})),
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1479
|
+
catch (e) {
|
|
1480
|
+
res.status(500).json({ error: e.message });
|
|
1481
|
+
}
|
|
1482
|
+
});
|
|
1389
1483
|
// Serve codebase timeline snapshots (AST Time-Travel Telemetry)
|
|
1484
|
+
// Snapshots come from watching this daemon's own cwd. Like onboarding, decline a caller from
|
|
1485
|
+
// another project instead of serving this project's history as theirs.
|
|
1390
1486
|
this.app.get('/api/project/timeline', (req, res) => {
|
|
1487
|
+
const expectedProject = typeof req.query.expectedProject === 'string' ? req.query.expectedProject : null;
|
|
1488
|
+
if ((0, ProjectIdentity_1.isProjectMismatch)(expectedProject, this.projectName)) {
|
|
1489
|
+
return res.json({ success: false, mismatch: true, servingProject: this.projectName, timeline: [] });
|
|
1490
|
+
}
|
|
1391
1491
|
res.json({ success: true, timeline: this.timeline });
|
|
1392
1492
|
});
|
|
1393
1493
|
// Dynamic Zero-Shot Onboarding mental model builder
|
|
@@ -1566,36 +1666,15 @@ class LemmaServer {
|
|
|
1566
1666
|
tk = Math.floor(Math.max(0, charsBefore - charsAfter) / 4);
|
|
1567
1667
|
SavingsLedger_1.savingsLedger.record(source, charsBefore, charsAfter);
|
|
1568
1668
|
}
|
|
1569
|
-
//
|
|
1669
|
+
// A saving reported by an MCP tool is not a proxy completion: it used to bump the
|
|
1670
|
+
// completion stats (total++, a "miss" for every non-cache source, a mock 15ms latency),
|
|
1671
|
+
// which skewed the proxy's hit rate, request count and average latency. Ledger only;
|
|
1672
|
+
// the event just shows the saving in the activity feed, with nothing invented.
|
|
1570
1673
|
if (tk > 0) {
|
|
1571
|
-
if (!this.stats[this.projectName]) {
|
|
1572
|
-
this.stats[this.projectName] = { total: 0, hits: 0, misses: 0, totalLatency: 0, totalTokensSaved: 0, providers: {} };
|
|
1573
|
-
}
|
|
1574
|
-
const s = this.stats[this.projectName];
|
|
1575
|
-
s.total++;
|
|
1576
|
-
s.totalTokensSaved += tk;
|
|
1577
|
-
s.totalLatency += 15; // mock latency for local MCP metrics
|
|
1578
|
-
if (source === 'cache') {
|
|
1579
|
-
s.hits++;
|
|
1580
|
-
}
|
|
1581
|
-
else {
|
|
1582
|
-
s.misses++;
|
|
1583
|
-
}
|
|
1584
|
-
if (!s.providers['mcp'])
|
|
1585
|
-
s.providers['mcp'] = { hits: 0, misses: 0 };
|
|
1586
|
-
if (source === 'cache')
|
|
1587
|
-
s.providers['mcp'].hits++;
|
|
1588
|
-
else
|
|
1589
|
-
s.providers['mcp'].misses++;
|
|
1590
|
-
// Write updated stats back to disk immediately
|
|
1591
|
-
writeJson(ProjectStore_1.STATS_FILE, this.stats);
|
|
1592
|
-
// Push to live telemetry logEvent array so the Dashboard event timeline updates instantly!
|
|
1593
1674
|
logEvent({
|
|
1594
|
-
type: source === 'cache' ? '
|
|
1595
|
-
project: this.projectName,
|
|
1596
|
-
latency: 15,
|
|
1675
|
+
type: source === 'cache' ? 'brain_hit' : source,
|
|
1597
1676
|
provider: 'mcp',
|
|
1598
|
-
tokens: tk
|
|
1677
|
+
tokens: tk,
|
|
1599
1678
|
});
|
|
1600
1679
|
}
|
|
1601
1680
|
res.json({ ok: true, snapshot: SavingsLedger_1.savingsLedger.getSnapshot().total });
|
|
@@ -1645,7 +1724,9 @@ class LemmaServer {
|
|
|
1645
1724
|
result: event.result || 'UNKNOWN',
|
|
1646
1725
|
latency: event.latency || 0,
|
|
1647
1726
|
tokensImpact: event.tokensImpact || 0,
|
|
1648
|
-
error: event.error
|
|
1727
|
+
error: typeof event.error === 'string' ? event.error.slice(0, 200) : null,
|
|
1728
|
+
receipt: typeof event.receipt === 'string' ? event.receipt : undefined,
|
|
1729
|
+
projectPath: typeof event.projectPath === 'string' ? event.projectPath : undefined,
|
|
1649
1730
|
timestamp: event.timestamp || new Date().toISOString(),
|
|
1650
1731
|
};
|
|
1651
1732
|
// Add to EVENT_LOG
|
|
@@ -1850,6 +1931,12 @@ class LemmaServer {
|
|
|
1850
1931
|
const tier = pro ? 'pro' : 'standard';
|
|
1851
1932
|
const isStream = !!req.body.stream;
|
|
1852
1933
|
let historyTokensSaved = 0;
|
|
1934
|
+
// The daemon is one process for every project on the machine; its own projectName is just
|
|
1935
|
+
// whichever cwd it was started from. Attribute each completion — stats, and above all the
|
|
1936
|
+
// semantic-cache collection — to the project the request comes from (MCP handshake
|
|
1937
|
+
// registry, then the messages themselves), so one project's cached answer is never served
|
|
1938
|
+
// to another. Falls back to the daemon's project when nothing identifies the caller.
|
|
1939
|
+
const requestProject = this.detectActiveProject(req)?.name ?? this.projectName;
|
|
1853
1940
|
if ((0, DebuggerState_1.isDebuggerPaused)()) {
|
|
1854
1941
|
res.setHeader('Retry-After', '5');
|
|
1855
1942
|
return res.status(503).json({ error: { message: 'Lemma is paused from the dashboard debugger. Resume it to continue.' } });
|
|
@@ -1865,15 +1952,17 @@ class LemmaServer {
|
|
|
1865
1952
|
return res.status(400).json({ error: 'Invalid request format' });
|
|
1866
1953
|
const { maskedPrompt: prompt, tokenMap } = semanticScrubber.mask(rawPrompt);
|
|
1867
1954
|
if (Object.keys(tokenMap).length > 0) {
|
|
1868
|
-
logEvent({ type: 'privacy:mask', count: Object.keys(tokenMap).length, project:
|
|
1955
|
+
logEvent({ type: 'privacy:mask', count: Object.keys(tokenMap).length, project: requestProject });
|
|
1956
|
+
SECRETS_MASKED.count += Object.keys(tokenMap).length;
|
|
1869
1957
|
}
|
|
1870
1958
|
const originalModel = req.body.model || 'gpt-4o';
|
|
1871
1959
|
// ── 1. Cache-First Hot Path ──
|
|
1872
1960
|
// Perform exact cache check immediately (resolves in <1ms)
|
|
1873
1961
|
const hit = cacheGet(provider, prompt);
|
|
1874
1962
|
if (hit) {
|
|
1875
|
-
|
|
1876
|
-
|
|
1963
|
+
// Measured from the cached response, like the semantic path — a flat 2000 used to go
|
|
1964
|
+
// straight into the ledger as if it had been counted.
|
|
1965
|
+
await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(hit.data));
|
|
1877
1966
|
const unmaskedData = semanticScrubber.unmask(hit.data, tokenMap);
|
|
1878
1967
|
if (isStream)
|
|
1879
1968
|
return SseRelay_1.SseRelay.simulateHit(res, unmaskedData, { provider: provider, tier, similarity: 1.0 });
|
|
@@ -1884,10 +1973,10 @@ class LemmaServer {
|
|
|
1884
1973
|
}
|
|
1885
1974
|
// Perform semantic cache check (resolves in ~15ms)
|
|
1886
1975
|
if (pro) {
|
|
1887
|
-
const semHit = await semanticGet(provider, prompt,
|
|
1976
|
+
const semHit = await semanticGet(provider, prompt, requestProject);
|
|
1888
1977
|
if (semHit) {
|
|
1889
1978
|
if (semHit.similarity >= 0.90) {
|
|
1890
|
-
await recordStat(this.stats,
|
|
1979
|
+
await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(semHit.data));
|
|
1891
1980
|
const unmaskedData = semanticScrubber.unmask(semHit.data, tokenMap);
|
|
1892
1981
|
const cacheHeader = semHit.hiveMind ? 'HIT-HIVE-MIND' : 'HIT';
|
|
1893
1982
|
if (isStream)
|
|
@@ -1938,7 +2027,7 @@ Adjusted Answer:`;
|
|
|
1938
2027
|
else if (provider === 'anthropic') {
|
|
1939
2028
|
synthesizedData.content[0].text = adjustedText;
|
|
1940
2029
|
}
|
|
1941
|
-
await recordStat(this.stats,
|
|
2030
|
+
await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(synthesizedData));
|
|
1942
2031
|
const unmaskedData = semanticScrubber.unmask(synthesizedData, tokenMap);
|
|
1943
2032
|
if (isStream) {
|
|
1944
2033
|
return SseRelay_1.SseRelay.simulateHit(res, unmaskedData, { provider: provider, tier, similarity: semHit.similarity });
|
|
@@ -1958,7 +2047,7 @@ Adjusted Answer:`;
|
|
|
1958
2047
|
const cloudHit = await cloudSync.get(prompt);
|
|
1959
2048
|
if (cloudHit) {
|
|
1960
2049
|
cacheSet(provider, prompt, cloudHit.data);
|
|
1961
|
-
await recordStat(this.stats,
|
|
2050
|
+
await recordStat(this.stats, requestProject, true, Date.now() - t0, provider, measureCachedTokens(cloudHit.data));
|
|
1962
2051
|
const unmaskedData = semanticScrubber.unmask(cloudHit.data, tokenMap);
|
|
1963
2052
|
if (isStream)
|
|
1964
2053
|
return SseRelay_1.SseRelay.simulateHit(res, unmaskedData, { provider: provider, tier, similarity: cloudHit.similarity || 0.95 });
|
|
@@ -2054,7 +2143,7 @@ Adjusted Answer:`;
|
|
|
2054
2143
|
}
|
|
2055
2144
|
}
|
|
2056
2145
|
if (isStream) {
|
|
2057
|
-
await recordStat(this.stats,
|
|
2146
|
+
await recordStat(this.stats, requestProject, false, Date.now() - t0, provider, historyTokensSaved);
|
|
2058
2147
|
if (!pro) {
|
|
2059
2148
|
await usageWriteQueue.enqueue(async () => {
|
|
2060
2149
|
const u = await loadUsage();
|
|
@@ -2068,7 +2157,7 @@ Adjusted Answer:`;
|
|
|
2068
2157
|
const data = await this.callUpstream(req.body, provider);
|
|
2069
2158
|
cacheSet(provider, prompt, data);
|
|
2070
2159
|
if (pro) {
|
|
2071
|
-
await semanticSet(provider, prompt, data,
|
|
2160
|
+
await semanticSet(provider, prompt, data, requestProject);
|
|
2072
2161
|
await cloudSync.set(prompt, data);
|
|
2073
2162
|
}
|
|
2074
2163
|
if (!pro) {
|
|
@@ -2078,7 +2167,7 @@ Adjusted Answer:`;
|
|
|
2078
2167
|
await writeJson(ProjectStore_1.USAGE_FILE, u);
|
|
2079
2168
|
});
|
|
2080
2169
|
}
|
|
2081
|
-
await recordStat(this.stats,
|
|
2170
|
+
await recordStat(this.stats, requestProject, false, Date.now() - t0, provider, historyTokensSaved);
|
|
2082
2171
|
return res.json(semanticScrubber.unmask(data, tokenMap));
|
|
2083
2172
|
}
|
|
2084
2173
|
catch (err) {
|
|
@@ -2230,23 +2319,31 @@ Adjusted Answer:`;
|
|
|
2230
2319
|
console.log('simply copy-paste this command into your MCP settings:');
|
|
2231
2320
|
console.log(`\n \x1b[1m\x1b[33mlemma mcp\x1b[0m\n`);
|
|
2232
2321
|
console.log('────────────────────────────────────────────────\n');
|
|
2233
|
-
//
|
|
2234
|
-
|
|
2235
|
-
|
|
2322
|
+
// The clipboard watcher runs as its own background process (startBackgroundClipboardWatcher,
|
|
2323
|
+
// tracked in CLIPBOARD_PID_FILE and stopped by `lemma stop`). This used to also start a
|
|
2324
|
+
// second, in-process watcher here: two pollers on one clipboard squeezed and reported the
|
|
2325
|
+
// same copy twice.
|
|
2326
|
+
if (process.env.LEMMA_DISABLE_CLIPBOARD === 'true') {
|
|
2236
2327
|
console.log(`📋 Clipboard Watcher : \x1b[90m[DISABLED via LEMMA_DISABLE_CLIPBOARD]\x1b[0m`);
|
|
2237
2328
|
}
|
|
2238
|
-
else {
|
|
2239
|
-
console.log(`📋 Clipboard Watcher : \x1b[33m[ACTIVE]\x1b[0m — polling clipboard every 1s (disable: LEMMA_DISABLE_CLIPBOARD=true)`);
|
|
2240
|
-
try {
|
|
2241
|
-
const { startClipboardWatcher } = require('../utils/ClipboardWatcher');
|
|
2242
|
-
startClipboardWatcher(1000);
|
|
2243
|
-
}
|
|
2244
|
-
catch (err) {
|
|
2245
|
-
console.warn(`⚠️ Could not auto-initialize Clipboard Watcher: ${err.message}`);
|
|
2246
|
-
}
|
|
2247
|
-
}
|
|
2248
2329
|
});
|
|
2249
|
-
|
|
2330
|
+
// A SIGTERM listener replaces Node's default exit, and server.close() only stops accepting:
|
|
2331
|
+
// keep-alive and socket.io connections, file watchers and intervals keep the loop alive, so
|
|
2332
|
+
// `lemma stop` used to leave a daemon with no port that never exited. Flush the Brain's
|
|
2333
|
+
// debounced write (same reason as the MCP bootstrap), close, and exit — with a hard cap.
|
|
2334
|
+
const shutdown = () => {
|
|
2335
|
+
try {
|
|
2336
|
+
(0, TheBrainV2_1.getBrain)().flushSync();
|
|
2337
|
+
}
|
|
2338
|
+
catch {
|
|
2339
|
+
// A flush failure must never block shutdown — the next save() heals.
|
|
2340
|
+
}
|
|
2341
|
+
server.close(() => process.exit(0));
|
|
2342
|
+
server.closeAllConnections();
|
|
2343
|
+
setTimeout(() => process.exit(0), 3000).unref();
|
|
2344
|
+
};
|
|
2345
|
+
process.once('SIGTERM', shutdown);
|
|
2346
|
+
process.once('SIGINT', shutdown);
|
|
2250
2347
|
}
|
|
2251
2348
|
}
|
|
2252
2349
|
// ── CLI ────────────────────────────────────────────────────────────────────────
|
|
@@ -2941,6 +3038,20 @@ commander_1.program.command('clipboard')
|
|
|
2941
3038
|
.option('--interval <ms>', 'Polling interval in milliseconds', '1000')
|
|
2942
3039
|
.action((opts) => {
|
|
2943
3040
|
try {
|
|
3041
|
+
// One watcher per machine: every poller on the same clipboard squeezes and reports the
|
|
3042
|
+
// same copy again. Defer to a live watcher recorded in the pid file; otherwise record us.
|
|
3043
|
+
try {
|
|
3044
|
+
const recorded = parseInt(fs_1.default.readFileSync(ProjectStore_1.CLIPBOARD_PID_FILE, 'utf8'), 10);
|
|
3045
|
+
if (recorded && recorded !== process.pid) {
|
|
3046
|
+
process.kill(recorded, 0); // throws if it's gone
|
|
3047
|
+
console.log(`✅ [Clipboard] Watcher already running (PID: ${recorded}) — not starting another.`);
|
|
3048
|
+
process.exit(0);
|
|
3049
|
+
}
|
|
3050
|
+
}
|
|
3051
|
+
catch {
|
|
3052
|
+
// No pid file, or its process is dead: this one is the watcher.
|
|
3053
|
+
}
|
|
3054
|
+
fs_1.default.writeFileSync(ProjectStore_1.CLIPBOARD_PID_FILE, String(process.pid));
|
|
2944
3055
|
const interval = parseInt(opts.interval) || 1000;
|
|
2945
3056
|
const { startClipboardWatcher } = require('../utils/ClipboardWatcher');
|
|
2946
3057
|
startClipboardWatcher(interval);
|