@nxuss/lemma 1.7.0 → 1.7.2

package/dist/esm/mcp/tools.js

@@ -5,40 +5,40 @@ import { execSync, spawnSync } from "child_process";
 import os from "os";
 import axios from "axios";
 import * as ts from "typescript";
-import { SemanticScrubber } from "../security/SemanticScrubber";
-import { ComplexityRouter } from "../proxy/ComplexityRouter";
-import { squeezeCode, squeezePrompt, pruneHistoryMessages, generateAstDiff } from "../utils/ContextSqueezer";
-import { reportSavings, reportCost } from "../utils/reportSavings";
-import { getProxyPort, safeResolvePath, logError, logWarn, estimateTokensFromResult, isCommandAllowed, loadAllowedCommands } from "./utils";
-import { computeFingerprint } from "../utils/FileFingerprint";
-import { analyzeBlameHeat } from "../utils/GitBlameHeat";
-import { analyzeTypeCoverage } from "../utils/TypeCoverage";
-import { buildImportTree } from "../utils/ImportTreeContext";
-import { readTokenBudgeted } from "../utils/TokenBudgetedReader";
-import { bulkFileDigest } from "../utils/BulkFileDigest";
-import { autoContextBundle } from "../utils/AutoContextBundle";
-import { checkRepeat, recordOutput } from "../utils/RepeatGuard";
-import { saveCheckpoint, loadLatestCheckpoint, buildCheckpointSummary } from "../utils/ConversationCheckpoint";
-import { runTestAutofix } from "../utils/TestAutofixInterceptor";
-import { findPattern, storePattern, getPatternStats } from "../utils/PromptPatternCache";
-import { runAnalysis, parseDiff } from "../pr-review/PRReviewEngine";
-import { ReviewStore } from "../pr-review/ReviewStore";
-import { runPRReview } from "../pr-review/reviewRunner";
-import { getBrain, deriveProjectId } from "../subconscious/TheBrainV2";
-import { collapseContext } from "../utils/ContextCollapser";
-import { getSymbolSurgicalContext } from "../utils/SymbolSurgicalContext";
-import { compressWormhole } from "../utils/WormholeCompressor";
-import { generateExecutiveReport } from "../utils/ExecutiveReport";
-import { surgicalASTInsert } from "../utils/SurgicalASTInsert";
-import { getInfraToolDefinitions, getInfraToolHandlers, INFRA_TOOL_NAMES } from "../infra/mcp-tools";
-import { resolveToolSurface, buildToolboxCatalog } from "./tool-profiles";
-import { lookupStateHash, storeStateHash } from "../utils/StateHashCache";
-import { recordReceiptEvent, getReceiptSummary } from "../utils/TokenReceipt";
-import { distillCommandOutput, buildDistillFooter, readRegion } from "../utils/CommandOutputDistiller";
-import { findMatch, reindentReplacement } from "../utils/PatchMatcher";
-import { searchWorkspace, groupSearchResults, parseExtensionFilter } from "../utils/WorkspaceSearch";
-import { checkAlreadySent, recordSent } from "../utils/ReadWorkspaceCache";
-import { getCachedSearch, setCachedSearch } from "../utils/SearchCache";
+import { SemanticScrubber } from "../security/SemanticScrubber.js";
+import { ComplexityRouter } from "../proxy/ComplexityRouter.js";
+import { squeezeCode, squeezePrompt, pruneHistoryMessages, generateAstDiff } from "../utils/ContextSqueezer.js";
+import { reportSavings, reportCost } from "../utils/reportSavings.js";
+import { getProxyPort, safeResolvePath, logError, logWarn, estimateTokensFromResult, isCommandAllowed, loadAllowedCommands } from "./utils.js";
+import { computeFingerprint } from "../utils/FileFingerprint.js";
+import { analyzeBlameHeat } from "../utils/GitBlameHeat.js";
+import { analyzeTypeCoverage } from "../utils/TypeCoverage.js";
+import { buildImportTree } from "../utils/ImportTreeContext.js";
+import { readTokenBudgeted } from "../utils/TokenBudgetedReader.js";
+import { bulkFileDigest } from "../utils/BulkFileDigest.js";
+import { autoContextBundle } from "../utils/AutoContextBundle.js";
+import { checkRepeat, recordOutput } from "../utils/RepeatGuard.js";
+import { saveCheckpoint, loadLatestCheckpoint, buildCheckpointSummary } from "../utils/ConversationCheckpoint.js";
+import { runTestAutofix } from "../utils/TestAutofixInterceptor.js";
+import { findPattern, storePattern, getPatternStats } from "../utils/PromptPatternCache.js";
+import { runAnalysis, parseDiff } from "../pr-review/PRReviewEngine.js";
+import { ReviewStore } from "../pr-review/ReviewStore.js";
+import { runPRReview } from "../pr-review/reviewRunner.js";
+import { getBrain, deriveProjectId } from "../subconscious/TheBrainV2.js";
+import { collapseContext } from "../utils/ContextCollapser.js";
+import { getSymbolSurgicalContext } from "../utils/SymbolSurgicalContext.js";
+import { compressWormhole } from "../utils/WormholeCompressor.js";
+import { generateExecutiveReport } from "../utils/ExecutiveReport.js";
+import { surgicalASTInsert } from "../utils/SurgicalASTInsert.js";
+import { getInfraToolDefinitions, getInfraToolHandlers, INFRA_TOOL_NAMES } from "../infra/mcp-tools.js";
+import { resolveToolSurface, buildToolboxCatalog } from "./tool-profiles.js";
+import { lookupStateHash, storeStateHash } from "../utils/StateHashCache.js";
+import { recordReceiptEvent, getReceiptSummary, getLedgerLength } from "../utils/TokenReceipt.js";
+import { distillCommandOutput, buildDistillFooter, readRegion } from "../utils/CommandOutputDistiller.js";
+import { findMatch, reindentReplacement } from "../utils/PatchMatcher.js";
+import { searchWorkspace, groupSearchResults, parseExtensionFilter } from "../utils/WorkspaceSearch.js";
+import { checkAlreadySent, recordSent } from "../utils/ReadWorkspaceCache.js";
+import { getCachedSearch, setCachedSearch } from "../utils/SearchCache.js";
 // ── Pro License Gate ──────────────────────────────────────────────────────────
 const LICENSE_FILE = path.join(os.homedir(), '.lemma-cache', 'license.json');
 let _proStatusCache = null;
@@ -103,6 +103,11 @@ const toolDefinitions = [
             properties: {
                 query: { type: "string", description: "The natural language query" },
                 limit: { type: "number", description: "Maximum results to return", default: 5 },
+                minSimilarity: {
+                    type: "number",
+                    description: "Similarity floor (0.0-1.0) a memory must clear to be returned. Lowering this surfaces loosely related memories that are usually noise — leave it alone unless a known-relevant memory is being filtered out.",
+                    default: 0.75,
+                },
             },
             required: ["query"],
         },
@@ -1156,6 +1161,41 @@ async function handleToolbox(args) {
     }
     throw new Error(`Unknown action: ${action}. Use 'list', 'schema', or 'call'.`);
 }
+/** Tools whose whole job is to put file contents into the model's context. */
+const FILE_READ_TOOLS = new Set([
+    "read_workspace_file",
+    "smart_file_slice",
+    "get_symbol_surgical_context",
+    "get_ast_hologram",
+    "list_workspace_dir",
+    "search_workspace",
+    "read_token_budgeted",
+    "bulk_file_digest",
+    "import_tree_context",
+]);
+function receiptTypeForTool(name) {
+    return FILE_READ_TOOLS.has(name) ? "file_read" : "tool_call";
+}
+/**
+ * A receipt entry is only useful if it says what the call was about. Pull the most
+ * identifying argument without dragging whole file contents into the ledger.
+ */
+function receiptLabelMeta(name, args) {
+    if (!args)
+        return {};
+    const meta = {};
+    if (typeof args.filePath === "string")
+        meta.filePath = args.filePath;
+    if (typeof args.dirPath === "string")
+        meta.dirPath = args.dirPath;
+    if (typeof args.query === "string")
+        meta.query = args.query.substring(0, 100);
+    if (typeof args.command === "string")
+        meta.command = args.command.substring(0, 100);
+    if (typeof args.symbolName === "string")
+        meta.symbolName = args.symbolName;
+    return meta;
+}
 export function setupToolsHandlers(server, onToolCall) {
     server.setRequestHandler(ListToolsRequestSchema, async () => ({
         tools: toolDefinitionsArray,
@@ -1190,7 +1230,16 @@ export function setupToolsHandlers(server, onToolCall) {
             throw new Error(`Unknown tool: ${name}`);
         }
         try {
+            // Sampled around the handler so a tool that logs its own, more specific event
+            // (a cache hit, a miss) isn't double-counted by the generic entry below.
+            const ledgerBefore = getLedgerLength();
             const result = await handler((args || {}));
+            if (getLedgerLength() === ledgerBefore) {
+                recordReceiptEvent(receiptTypeForTool(name), name, {
+                    tool: name,
+                    ...receiptLabelMeta(name, args),
+                });
+            }
             const tokensImpact = estimateTokensFromResult(result);
             onToolCall?.({
                 tool: name,
@@ -1225,16 +1274,35 @@ async function handleScrubPrivacy(args) {
     const { maskedPrompt } = scrubber.mask(text);
     return { content: [{ type: "text", text: maskedPrompt }] };
 }
+/**
+ * Minimum similarity for a stored memory to be offered for reuse. Shared by
+ * search_memory and smarter_cache so the two never disagree about whether the same
+ * query is a hit.
+ */
+const DEFAULT_MEMORY_SIMILARITY_FLOOR = 0.75;
 async function handleSearchMemory(args) {
     const query = args?.query;
     const limit = args?.limit || 5;
+    // A floor of 0 returns the nearest neighbour no matter how unrelated it is, and the
+    // formatting below then presents it as a reusable memory. Match smarter_cache's
+    // threshold so both paths agree on what counts as a hit.
+    const minSimilarity = typeof args?.minSimilarity === "number" ? args.minSimilarity : DEFAULT_MEMORY_SIMILARITY_FLOOR;
     if (!query)
         throw new Error("Query is required");
     try {
         const brain = getBrain();
-        const results = brain.search(query, limit, 0, { projectId: deriveProjectId() });
+        const results = brain.search(query, limit, minSimilarity, { projectId: deriveProjectId() });
         if (results.length === 0) {
-            return { content: [{ type: "text", text: "No relevant memories found in Lemma's Brain." }] };
+            recordReceiptEvent("reasoning", query.substring(0, 100), {
+                tool: "search_memory",
+                reason: `no memory above the ${minSimilarity} similarity floor`,
+            });
+            return {
+                content: [{
+                        type: "text",
+                        text: `No memory in Lemma's Brain scored at or above the ${(minSimilarity * 100).toFixed(0)}% similarity floor for this query. Investigate from scratch — there is nothing safe to reuse.`,
+                    }],
+            };
         }
         const fresh = results.filter((r) => r.fresh);
         const stale = results.filter((r) => !r.fresh);
@@ -1245,6 +1313,17 @@ async function handleSearchMemory(args) {
             const bestFresh = fresh[0];
             const tokensSaved = Math.max(100, Math.floor(String(bestFresh.response).length / 4));
             reportSavings({ source: "cache", tokens: tokensSaved, toolName: "search_memory", query: query.substring(0, 100) });
+            recordReceiptEvent("semantic_cache_hit", query.substring(0, 100), {
+                tool: "search_memory",
+                similarity: bestFresh.similarity,
+                tokensSaved,
+            });
+        }
+        else {
+            recordReceiptEvent("reasoning", query.substring(0, 100), {
+                tool: "search_memory",
+                reason: `${stale.length} similar memory/memories found but all stale — must re-verify`,
+            });
         }
         const parts = [];
         if (fresh.length > 0) {
@@ -1273,12 +1352,13 @@ async function handleStoreMemory(args) {
     try {
         const brain = getBrain();
         const storeRes = brain.store(query, responseText, provider, 0.92, filePaths);
-        const tokensSaved = Math.max(100, Math.floor(responseText.length / 4));
-        reportSavings({
-            source: "cache",
-            tokens: tokensSaved,
-            toolName: "store_memory",
-            query: query.substring(0, 100),
+        // Storing a memory saves nothing — it only creates the chance of a saving later.
+        // Crediting tokens here inflated the ledger on write and then credited the same
+        // answer again on every read. The saving is booked by search_memory on a fresh hit.
+        recordReceiptEvent("tool_call", `store_memory: ${query.substring(0, 100)}`, {
+            tool: "store_memory",
+            filePaths,
+            note: "memory written — no tokens saved yet",
         });
         const trackingNote = filePaths && filePaths.length > 0 ? ` Tracking freshness against ${filePaths.length} file(s) — this memory auto-invalidates if they change.` : "";
         return { content: [{ type: "text", text: `Success: Memory stored. ${storeRes.reason}${trackingNote}` }] };
@@ -1314,7 +1394,7 @@ async function handleGetRoutingAdvice(args) {
 async function handleAutoHeal(args) {
     const apply = !!args?.apply;
     try {
-        const { performAutoHeal: heal } = await import("../cli/lemma-proxy");
+        const { performAutoHeal: heal } = await import("../cli/lemma-proxy.js");
         const res = await heal(apply);
         return { content: [{ type: "text", text: JSON.stringify(res, null, 2) }] };
     }
@@ -1800,7 +1880,7 @@ async function handleGetProjectOnboarding(_args) {
         ],
     };
 }
-function extractSymbolsWithTsCompiler(filePath, relPath) {
+function extractSymbolsWithTsCompiler(filePath, relPath, parseErrors) {
     try {
         const src = fs.readFileSync(filePath, "utf8");
         const sourceFile = ts.createSourceFile(filePath, src, ts.ScriptTarget.Latest, true);
@@ -1808,7 +1888,10 @@ function extractSymbolsWithTsCompiler(filePath, relPath) {
         function visit(node) {
             const mods = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined;
             const isExport = mods?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword);
-            if (isExport || ts.isSourceFile(node.parent)) {
+            // node.parent is undefined on the SourceFile root itself, and ts.isSourceFile()
+            // dereferences .kind without a guard — calling it unguarded threw on the very
+            // first visit(), so the catch below swallowed it and every file returned [].
+            if (isExport || (node.parent && ts.isSourceFile(node.parent))) {
                 if (ts.isFunctionDeclaration(node) && node.name) {
                     symbols.push({
                         kind: "function",
@@ -1877,10 +1960,11 @@ function extractSymbolsWithTsCompiler(filePath, relPath) {
     }
     catch (err) {
         logWarn("get_ast_hologram", `Could not parse ${relPath}: ${err}`);
+        parseErrors?.push(`${relPath}: ${err}`);
         return [];
     }
 }
-function walkDirForHologram(dir, rel, extensions) {
+function walkDirForHologram(dir, rel, extensions, stats) {
     let allSymbols = [];
     try {
         const entries = fs.readdirSync(dir, { withFileTypes: true });
@@ -1890,18 +1974,20 @@ function walkDirForHologram(dir, rel, extensions) {
             const fullPath = path.join(dir, entry.name);
             const relPath = rel ? path.join(rel, entry.name) : entry.name;
             if (entry.isDirectory()) {
-                allSymbols = allSymbols.concat(walkDirForHologram(fullPath, relPath, extensions));
+                allSymbols = allSymbols.concat(walkDirForHologram(fullPath, relPath, extensions, stats));
             }
             else {
                 const ext = entry.name.split(".").pop() || "";
                 if (extensions.includes(ext)) {
-                    allSymbols = allSymbols.concat(extractSymbolsWithTsCompiler(fullPath, relPath));
+                    stats.filesScanned++;
+                    allSymbols = allSymbols.concat(extractSymbolsWithTsCompiler(fullPath, relPath, stats.parseErrors));
                 }
             }
         }
     }
     catch (err) {
         logWarn("get_ast_hologram", `Error walking dir ${dir}: ${err}`);
+        stats.parseErrors.push(`walk ${rel || "."}: ${err}`);
     }
     return allSymbols;
 }
@@ -1910,18 +1996,32 @@ async function handleGetAstHologram(args) {
     const extensions = args?.extensions || ["ts", "tsx", "js", "jsx"];
     const workspaceRoot = process.cwd();
     const { resolved } = safeResolvePath(workspaceRoot, dirPath);
-    const symbols = walkDirForHologram(resolved, dirPath, extensions);
+    const stats = { filesScanned: 0, parseErrors: [] };
+    const symbols = walkDirForHologram(resolved, dirPath, extensions, stats);
     const byFile = {};
     symbols.forEach((s) => {
         if (!byFile[s.file])
             byFile[s.file] = [];
         byFile[s.file].push({ kind: s.kind, name: s.name, line: s.line });
     });
+    // An empty map is reported as a failure, not as an answer: a silent "0 symbols" is
+    // indistinguishable from a working scan of an empty dir, which is how a parse bug
+    // stayed invisible here before. Say when nothing could be read.
+    const warning = stats.filesScanned === 0
+        ? `No files matching [${extensions.join(", ")}] were found under "${dirPath || "."}" — check the path and extensions.`
+        : symbols.length === 0
+            ? `Scanned ${stats.filesScanned} file(s) but extracted 0 symbols — this is very likely an extraction failure, not an empty codebase. Do not treat this as "the directory has no exports".`
+            : undefined;
     const hologram = {
         workspace: path.basename(workspaceRoot),
         scannedDir: dirPath || ".",
         totalSymbols: symbols.length,
+        filesScanned: stats.filesScanned,
         totalFiles: Object.keys(byFile).length,
+        ...(warning ? { warning } : {}),
+        ...(stats.parseErrors.length > 0
+            ? { parseErrors: stats.parseErrors.slice(0, 10), parseErrorCount: stats.parseErrors.length }
+            : {}),
         map: byFile,
     };
     return { content: [{ type: "text", text: JSON.stringify(hologram, null, 2) }] };
@@ -2264,7 +2364,7 @@ async function handleTurboSqueeze(args) {
     const ratio = text.length > 0 ? ((1 - squeezed.length / text.length) * 100).toFixed(1) : "0";
     const saved = Math.max(0, text.length - squeezed.length);
     try {
-        const { reportSavings } = await import("../utils/reportSavings");
+        const { reportSavings } = await import("../utils/reportSavings.js");
         reportSavings({ source: "contextSqueeze", charsBefore: text.length, charsAfter: squeezed.length, toolName: "turbosqueeze" });
     }
     catch { }
@@ -2309,7 +2409,7 @@ async function handleCompressContext(args) {
         ...recent,
     ];
     try {
-        const { reportSavings } = await import("../utils/reportSavings");
+        const { reportSavings } = await import("../utils/reportSavings.js");
         reportSavings({ source: "historyPrune", tokens: tokensSaved, toolName: "compress_context" });
     }
     catch { }
@@ -2324,7 +2424,7 @@ async function handleCompressContext(args) {
 async function handleSmarterCache(args) {
     const query = args?.query;
     const context = args?.context || "";
-    const threshold = typeof args?.threshold === "number" ? args.threshold : 0.75;
+    const threshold = typeof args?.threshold === "number" ? args.threshold : DEFAULT_MEMORY_SIMILARITY_FLOOR;
     if (!query)
         throw new Error("query is required");
     const fullQuery = context ? `${query}\n\nContext: ${context}` : query;
@@ -2336,7 +2436,7 @@ async function handleSmarterCache(args) {
             const responseText = topHit.response;
             const tokensSaved = Math.floor(responseText.length / 4);
             try {
-                const { reportSavings } = await import("../utils/reportSavings");
+                const { reportSavings } = await import("../utils/reportSavings.js");
                 reportSavings({ source: "cache", tokens: tokensSaved, toolName: "smarter_cache", query: query.substring(0, 100) });
             }
             catch { }