nodebench-mcp 2.17.0 → 2.18.0

package/dist/tools/toolRegistry.d.ts

@@ -105,6 +105,23 @@ export declare function hybridSearch(query: string, tools: Array<{
     explain?: boolean;
     /** Pre-computed query embedding vector for semantic search (passed from async caller) */
     embeddingQueryVec?: Float32Array;
+    /** If true, search ALL_REGISTRY_ENTRIES (full 175-tool registry) regardless of loaded preset.
+     *  Needed for dynamic loading: discover_tools must find unloaded tools to suggest load_toolset. */
+    searchFullRegistry?: boolean;
+    /** Ablation flags: disable individual strategies to measure their contribution */
+    ablation?: {
+        disableSynonyms?: boolean;
+        disableFuzzy?: boolean;
+        disableTagCoverage?: boolean;
+        disableTfIdf?: boolean;
+        disableNgram?: boolean;
+        disableBigram?: boolean;
+        disableDense?: boolean;
+        disableDomainBoost?: boolean;
+        disableTraceEdges?: boolean;
+        disablePrefix?: boolean;
+        disableEmbedding?: boolean;
+    };
 }): SearchResult[];
 /** Available search modes for discover_tools */
 export declare const SEARCH_MODES: SearchMode[];

package/dist/tools/toolRegistry.js

@@ -1097,7 +1097,7 @@ const REGISTRY_ENTRIES = [
     {
         name: "call_llm",
         category: "llm",
-        tags: ["llm", "call", "generate", "prompt", "gemini", "openai", "anthropic", "gpt", "claude"],
+        tags: ["llm", "call", "generate", "prompt", "gemini", "openai", "anthropic", "gpt", "claude", "model", "ai", "inference", "completion", "analyze", "text"],
         quickRef: {
             nextAction: "LLM response received. Validate output quality. Use for analysis, generation, or judgment tasks.",
             nextTools: ["extract_structured_data", "record_learning"],
@@ -1107,7 +1107,7 @@ const REGISTRY_ENTRIES = [
     {
         name: "extract_structured_data",
         category: "llm",
-        tags: ["extract", "structured", "data", "json", "parse", "schema", "llm"],
+        tags: ["extract", "structured", "data", "json", "parse", "schema", "llm", "model", "ai", "transform", "output"],
         quickRef: {
             nextAction: "Structured data extracted. Validate against expected schema. Use for downstream processing.",
             nextTools: ["record_eval_result", "record_learning"],
@@ -1117,7 +1117,7 @@ const REGISTRY_ENTRIES = [
     {
         name: "benchmark_models",
         category: "llm",
-        tags: ["benchmark", "models", "compare", "latency", "quality", "cost", "llm"],
+        tags: ["benchmark", "models", "compare", "latency", "quality", "cost", "llm", "ai", "gpt", "claude", "gemini", "evaluate"],
         quickRef: {
             nextAction: "Benchmark complete. Compare models on quality, latency, and cost. Record winner with record_learning.",
             nextTools: ["record_learning", "call_llm"],
@@ -2110,6 +2110,7 @@ export function getToolComplexity(toolName) {
 }
 // ── Synonym / semantic expansion map ──────────────────────────────────────
 const SYNONYM_MAP = {
+    // ── Existing technical synonyms ──
     verify: ["validate", "check", "confirm", "test", "assert", "ensure", "correct"],
     test: ["verify", "validate", "check", "assert", "spec", "expect"],
     search: ["find", "discover", "lookup", "query", "locate", "browse"],
@@ -2118,7 +2119,7 @@ const SYNONYM_MAP = {
     setup: ["bootstrap", "init", "configure", "scaffold", "create"],
     fix: ["resolve", "repair", "debug", "patch", "correct"],
     deploy: ["ship", "publish", "release", "launch", "ci", "cd", "pipeline"],
-    analyze: ["inspect", "review", "examine", "audit", "scan"],
+    analyze: ["inspect", "review", "examine", "audit", "scan", "screenshot"],
     monitor: ["watch", "observe", "track", "follow"],
     security: ["vulnerability", "audit", "cve", "secret", "credential", "leak", "exposure"],
     benchmark: ["measure", "evaluate", "score", "grade", "performance", "capability"],
@@ -2131,7 +2132,7 @@ const SYNONYM_MAP = {
     ui: ["frontend", "visual", "screenshot", "responsive", "layout", "css", "component"],
     llm: ["model", "ai", "generate", "prompt", "gpt", "claude", "gemini"],
     migrate: ["upgrade", "update", "port", "convert", "transition", "refactor"],
-    review: ["inspect", "audit", "pr", "pull-request", "feedback", "critique"],
+    review: ["inspect", "audit", "pr", "pull-request", "feedback", "critique", "merge"],
     performance: ["speed", "latency", "optimize", "fast", "slow", "bottleneck"],
     data: ["csv", "xlsx", "json", "pdf", "file", "parse", "extract", "spreadsheet"],
     paper: ["academic", "research", "write", "publish", "neurips", "icml", "arxiv", "section"],
@@ -2147,6 +2148,31 @@ const SYNONYM_MAP = {
     why: ["purpose", "reason", "intentionality", "motivation", "goal", "critter"],
     purpose: ["why", "reason", "intentionality", "motivation", "goal", "critter"],
     reflect: ["think", "pause", "reconsider", "intentionality", "metacognition", "critter"],
+    // ── New user natural language expansions (ablation-driven) ──
+    website: ["seo", "url", "web", "fetch", "page", "lighthouse", "performance"],
+    webpage: ["seo", "url", "web", "fetch", "page", "html"],
+    fast: ["seo", "performance", "speed", "latency", "lighthouse"],
+    slow: ["seo", "performance", "speed", "latency", "lighthouse", "bottleneck"],
+    inbox: ["email", "read_emails", "send_email", "messages"],
+    email: ["send_email", "read_emails", "inbox", "messages", "smtp", "imap"],
+    ai: ["llm", "model", "prompt", "generate", "gpt", "claude", "gemini", "call_llm"],
+    summarize: ["llm", "extract", "generate", "analyze", "call_llm"],
+    bugs: ["scan", "code", "analysis", "dependencies", "vulnerabilities", "debug"],
+    readme: ["documentation", "generate", "report", "markdown", "document"],
+    compiles: ["closed_loop", "build", "test", "verify", "compile"],
+    works: ["test", "verify", "closed_loop", "flywheel", "quality", "check"],
+    commits: ["git", "commit", "messages", "conventional", "pr"],
+    push: ["git", "commit", "merge", "pr", "deploy"],
+    merge: ["git", "pr", "review", "checklist", "enforce"],
+    open: ["read", "file", "csv", "json", "parse", "load"],
+    look: ["read", "analyze", "inspect", "view", "examine", "fetch"],
+    good: ["quality", "gate", "check", "validate", "analysis"],
+    screenshot: ["analyze", "capture", "vision", "ui", "responsive", "visual"],
+    run: ["test", "execute", "closed_loop", "quality", "cli"],
+    check: ["verify", "validate", "audit", "scan", "review", "gate", "test"],
+    help: ["generate", "create", "scaffold", "analyze", "recommend"],
+    computer: ["llm", "ai", "model", "analyze", "extract"],
+    text: ["extract", "parse", "read", "llm", "structured", "analyze"],
 };
 // ── TF-IDF: compute inverse document frequency for tags ───────────────────
 let _idfCache = null;
@@ -2458,6 +2484,7 @@ export function hybridSearch(query, tools, options) {
     const explain = options?.explain ?? false;
     const mode = options?.mode ?? "hybrid";
     const idf = computeIDF();
+    const ab = options?.ablation ?? {};
     // Regex mode: compile pattern, match against name+description
     let regexPattern = null;
     if (mode === "regex") {
@@ -2516,7 +2543,16 @@ export function hybridSearch(query, tools, options) {
         }
     }
     const toolScores = new Map();
-    for (const tool of tools) {
+    // When searchFullRegistry is enabled, search ALL registry entries (not just loaded tools).
+    // This lets discover_tools find unloaded tools and suggest load_toolset.
+    const toolDescMap = new Map(tools.map(t => [t.name, t.description]));
+    const searchList = options?.searchFullRegistry
+        ? ALL_REGISTRY_ENTRIES.map(e => ({
+            name: e.name,
+            description: toolDescMap.get(e.name) ?? `${e.tags.join(" ")} ${e.category} ${e.phase}`,
+        }))
+        : tools;
+    for (const tool of searchList) {
         const entry = TOOL_REGISTRY.get(tool.name);
         if (!entry)
             continue;
@@ -2557,7 +2593,7 @@ export function hybridSearch(query, tools, options) {
             }
         }
         // ── MODE: prefix ──
-        if (mode === "hybrid" || mode === "prefix") {
+        if ((mode === "hybrid" || mode === "prefix") && !ab.disablePrefix) {
             for (const word of queryWords) {
                 if (nameLower.startsWith(word)) {
                     score += 20;
@@ -2589,7 +2625,7 @@ export function hybridSearch(query, tools, options) {
                 // Tag exact match (weighted by TF-IDF)
                 if (entry.tags.includes(word)) {
                     const idfWeight = idf.get(word) ?? 3;
-                    const tagScore = Math.round(10 * (idfWeight / 3));
+                    const tagScore = ab.disableTfIdf ? 10 : Math.round(10 * (idfWeight / 3));
                     score += tagScore;
                     reasons.push(`keyword:tag(${word},idf=${idfWeight.toFixed(1)})`);
                 }
@@ -2614,9 +2650,21 @@ export function hybridSearch(query, tools, options) {
                 score += 12;
                 reasons.push(`keyword:methodology(${entry.quickRef.methodology})`);
             }
+            // ── TAG COVERAGE BONUS: reward tools where many query words hit tags ──
+            // If 60%+ of query words match tags, that's a strong relevance signal.
+            if (queryWords.length >= 3 && !ab.disableTagCoverage) {
+                const tagSet = new Set(entry.tags);
+                const hits = queryWords.filter(w => tagSet.has(w)).length;
+                const coverage = hits / queryWords.length;
+                if (coverage >= 0.6) {
+                    const coverageBonus = Math.round(coverage * hits * 5);
+                    score += coverageBonus;
+                    reasons.push(`tag_coverage:${hits}/${queryWords.length}(${(coverage * 100).toFixed(0)}%,+${coverageBonus})`);
+                }
+            }
         }
         // ── SEMANTIC: synonym expansion (only score expanded words, not original) ──
-        if (mode === "hybrid" || mode === "semantic") {
+        if ((mode === "hybrid" || mode === "semantic") && !ab.disableSynonyms) {
             for (const syn of expandedWords) {
                 if (queryWords.includes(syn))
                     continue; // skip original words
@@ -2635,7 +2683,7 @@ export function hybridSearch(query, tools, options) {
             }
         }
         // ── FUZZY: Levenshtein distance for typo tolerance ──
-        if (mode === "hybrid" || mode === "fuzzy") {
+        if ((mode === "hybrid" || mode === "fuzzy") && !ab.disableFuzzy) {
             for (const word of queryWords) {
                 if (word.length < 4)
                     continue; // skip short words for fuzzy
@@ -2662,7 +2710,7 @@ export function hybridSearch(query, tools, options) {
             }
         }
         // ── N-GRAM: trigram similarity ──
-        if (mode === "hybrid" || mode === "fuzzy") {
+        if ((mode === "hybrid" || mode === "fuzzy") && !ab.disableNgram) {
             for (const word of queryWords) {
                 if (word.length < 4)
                     continue;
@@ -2683,7 +2731,7 @@ export function hybridSearch(query, tools, options) {
             }
         }
         // ── BIGRAM: phrase matching ──
-        if (queryBigrams.length > 0) {
+        if (queryBigrams.length > 0 && !ab.disableBigram) {
             for (const bigram of queryBigrams) {
                 if (allText.includes(bigram)) {
                     score += 15;
@@ -2692,7 +2740,7 @@ export function hybridSearch(query, tools, options) {
             }
         }
         // ── DENSE: TF-IDF cosine similarity (query vec pre-computed above) ──
-        if (denseQueryVec && denseDocVectors) {
+        if (denseQueryVec && denseDocVectors && !ab.disableDense) {
             const docVec = denseDocVectors.get(tool.name);
             if (docVec) {
                 const sim = cosineSimilarity(denseQueryVec, docVec);
@@ -2704,7 +2752,7 @@ export function hybridSearch(query, tools, options) {
             }
         }
         // ── EMBEDDING: Agent-as-a-Graph bipartite RRF (ranks pre-computed above) ──
-        if (embToolRanks && embDomainRanks) {
+        if (embToolRanks && embDomainRanks && !ab.disableEmbedding) {
             const toolRank = embToolRanks.get(tool.name);
             if (toolRank) {
                 const rrfScore = Math.round(WRRF_ALPHA_T * 1000 / (WRRF_K + toolRank));
@@ -2744,18 +2792,18 @@ export function hybridSearch(query, tools, options) {
             neighbors.forEach((n) => traceBoostTargets.add(n));
     }
     const results = [];
-    for (const tool of tools) {
+    for (const tool of searchList) {
         const entry = TOOL_REGISTRY.get(tool.name);
         const scored = toolScores.get(tool.name);
         if (!entry || !scored)
             continue;
-        const domainBoost = getDomainBoost(entry.category, topCategories);
+        const domainBoost = ab.disableDomainBoost ? 0 : getDomainBoost(entry.category, topCategories);
         if (domainBoost > 0) {
             scored.score += domainBoost;
             scored.reasons.push(`domain_boost:+${domainBoost}`);
         }
         // Execution trace edge: boost tools that frequently co-occur with top results
-        if (traceBoostTargets.has(tool.name) && !topToolNames.includes(tool.name)) {
+        if (traceBoostTargets.has(tool.name) && !topToolNames.includes(tool.name) && !ab.disableTraceEdges) {
             scored.score += TRACE_EDGE_BOOST;
             scored.reasons.push(`trace_edge:+${TRACE_EDGE_BOOST}`);
         }