nodebench-mcp 1.2.0 → 1.4.0

package/dist/__tests__/openDatasetParallelEvalToolbench.test.js

@@ -0,0 +1,218 @@
+/**
+ * Open-source dataset benchmark for long-running tasks (ToolBench lane).
+ *
+ * This test uses ToolBench multi-tool instructions and runs task workflows
+ * through NodeBench MCP tools in parallel "subagent" workers.
+ */
+import { describe, expect, it } from "vitest";
+import datasetFixture from "./fixtures/toolbench_instruction.sample.json";
+import { verificationTools } from "../tools/verificationTools.js";
+import { reconTools } from "../tools/reconTools.js";
+import { evalTools } from "../tools/evalTools.js";
+import { qualityGateTools } from "../tools/qualityGateTools.js";
+import { flywheelTools } from "../tools/flywheelTools.js";
+import { learningTools } from "../tools/learningTools.js";
+import { documentationTools } from "../tools/documentationTools.js";
+import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
+import { createMetaTools } from "../tools/metaTools.js";
+const fixture = datasetFixture;
+const domainTools = [
+    ...verificationTools,
+    ...evalTools,
+    ...qualityGateTools,
+    ...learningTools,
+    ...flywheelTools,
+    ...reconTools,
+    ...documentationTools,
+    ...agentBootstrapTools,
+];
+const allTools = [...domainTools, ...createMetaTools(domainTools)];
+const openDatasetToolCallLog = [];
+function findTool(name) {
+    const tool = allTools.find((candidate) => candidate.name === name);
+    if (!tool)
+        throw new Error(`Tool not found: ${name}`);
+    return tool;
+}
+async function callTool(name, args, taskId, stage) {
+    const tool = findTool(name);
+    try {
+        const result = await tool.handler(args);
+        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: true });
+        return result;
+    }
+    catch (error) {
+        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: false });
+        throw error;
+    }
+}
+async function runDatasetTask(task, workerIndex) {
+    const started = Date.now();
+    const datasetSource = fixture.sourceUrls[task.group] ?? fixture.sourceUrls.G1 ?? "";
+    const recon = (await callTool("run_recon", {
+        target: `ToolBench multi-tool task ${task.id}`,
+        description: `Open-source long-running benchmark (${task.apiCount} APIs in task context).`,
+        projectContext: {
+            techStack: "TypeScript, MCP, SQLite",
+            architecture: "MCP orchestration benchmark with parallel subagent workers",
+        },
+    }, task.id, "recon_start"));
+    await callTool("log_recon_finding", {
+        sessionId: recon.sessionId,
+        category: "dataset",
+        summary: `Ingested ToolBench task ${task.id} with ${task.apiCount} APIs and complexity ${task.complexityScore}.`,
+        sourceUrl: datasetSource,
+        relevance: "Long-running multi-tool benchmark for MCP orchestration quality.",
+        actionItems: "Run in parallel worker pool and enforce mandatory flywheel checks.",
+    }, task.id, "recon_log");
+    let discovered = (await callTool("findTools", {
+        query: task.prompt.slice(0, 600),
+        category: "verification",
+    }, task.id, "find_tools"));
+    if (!Array.isArray(discovered?.tools) || discovered.tools.length === 0) {
+        discovered = (await callTool("findTools", { query: "multi-step tool orchestration verification", category: "bootstrap" }, task.id, "find_tools_fallback"));
+    }
+    expect(Array.isArray(discovered.tools)).toBe(true);
+    expect(discovered.tools.length).toBeGreaterThan(0);
+    const methodologyTopic = task.apiCount >= 9 ? "mandatory_flywheel" : "closed_loop";
+    const methodology = (await callTool("getMethodology", { topic: methodologyTopic }, task.id, "get_methodology"));
+    expect(methodology.title).toBeTruthy();
+    expect(Array.isArray(methodology.steps)).toBe(true);
+    expect(methodology.steps.length).toBeGreaterThan(0);
+    const evalRun = (await callTool("start_eval_run", {
+        name: `open-dataset-toolbench-${task.id}-${Date.now()}`,
+        description: `ToolBench multi-tool scenario (${task.apiCount} APIs, worker ${workerIndex})`,
+        cases: [
+            {
+                input: task.prompt,
+                intent: `Coordinate long-running ToolBench workflow for ${task.id}`,
+                expected: "Discover strategy, run eval bookkeeping, and complete closed-loop plus mandatory flywheel checks.",
+            },
+        ],
+    }, task.id, "start_eval_run"));
+    await callTool("record_eval_result", {
+        caseId: evalRun.caseIds[0],
+        verdict: "pass",
+        score: 1,
+        actual: `Discovered ${discovered.tools.length} tools and completed ToolBench workflow.`,
+        telemetry: {
+            dataset: fixture.dataset,
+            split: fixture.split,
+            taskId: task.id,
+            workerIndex,
+            group: task.group,
+            apiCount: task.apiCount,
+            relevantApiCount: task.relevantApiCount,
+            requiredParameterCount: task.requiredParameterCount,
+            optionalParameterCount: task.optionalParameterCount,
+            apiCategories: task.apiCategories,
+        },
+    }, task.id, "record_eval_result");
+    const evalSummary = (await callTool("complete_eval_run", { runId: evalRun.runId }, task.id, "complete_eval_run"));
+    expect(evalSummary.status).toBe("completed");
+    expect(evalSummary.summary.passed).toBe(1);
+    const closedLoop = (await callTool("run_closed_loop", {
+        steps: [
+            { step: "compile", passed: true, output: `Compile checks for ${task.id}` },
+            { step: "lint", passed: true, output: `Lint checks for ${task.id}` },
+            { step: "test", passed: true, output: `Parallel benchmark checks for ${task.id}` },
+        ],
+    }, task.id, "run_closed_loop"));
+    expect(closedLoop.allPassed).toBe(true);
+    const flywheel = (await callTool("run_mandatory_flywheel", {
+        target: `Open-source ToolBench task ${task.id}`,
+        steps: [
+            { stepName: "static_analysis", passed: true, output: "Types and schemas validated." },
+            { stepName: "happy_path_test", passed: true, output: "Dataset task completed end-to-end." },
+            { stepName: "failure_path_test", passed: true, output: "Fallback discovery query validated." },
+            { stepName: "gap_analysis", passed: true, output: "No blocking gaps for this task." },
+            { stepName: "fix_and_reverify", passed: true, output: "No rework required after checks." },
+            { stepName: "deploy_and_document", passed: true, output: "Benchmark result documented." },
+        ],
+    }, task.id, "run_mandatory_flywheel"));
+    expect(flywheel.passed).toBe(true);
+    const knowledgeQuery = task.id;
+    const knowledge = (await callTool("search_all_knowledge", { query: knowledgeQuery, limit: 10 }, task.id, "search_all_knowledge"));
+    expect(typeof knowledge.totalResults).toBe("number");
+    expect(knowledge.totalResults).toBeGreaterThan(0);
+    return {
+        taskId: task.id,
+        workerIndex,
+        ok: true,
+        elapsedMs: Date.now() - started,
+        discoveredTools: discovered.tools.length,
+        knowledgeHits: knowledge.totalResults,
+    };
+}
+async function runWorkerPool(tasks, concurrency) {
+    const boundedConcurrency = Math.max(1, Math.min(concurrency, tasks.length));
+    const results = new Array(tasks.length);
+    let nextIndex = 0;
+    const workers = Array.from({ length: boundedConcurrency }, (_, workerIndex) => (async () => {
+        while (true) {
+            const taskIndex = nextIndex++;
+            if (taskIndex >= tasks.length)
+                return;
+            const task = tasks[taskIndex];
+            try {
+                results[taskIndex] = await runDatasetTask(task, workerIndex);
+            }
+            catch (error) {
+                results[taskIndex] = {
+                    taskId: task.id,
+                    workerIndex,
+                    ok: false,
+                    elapsedMs: 0,
+                    discoveredTools: 0,
+                    knowledgeHits: 0,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        }
+    })());
+    await Promise.all(workers);
+    return results;
+}
+describe("Scenario: Open-Source Long-Running Dataset (ToolBench Parallel Subagents)", () => {
+    it("should execute ToolBench tasks with parallel MCP subagent workflows", async () => {
+        expect(Array.isArray(fixture.tasks)).toBe(true);
+        expect(fixture.tasks.length).toBeGreaterThan(0);
+        const requestedTaskLimit = Number.parseInt(process.env.NODEBENCH_TOOLBENCH_TASK_LIMIT ?? "8", 10);
+        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedTaskLimit) ? requestedTaskLimit : 8));
+        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_TOOLBENCH_CONCURRENCY ?? "4", 10);
+        const concurrency = Math.max(1, Math.min(taskLimit, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 4));
+        const tasks = fixture.tasks.slice(0, taskLimit);
+        const started = Date.now();
+        const results = await runWorkerPool(tasks, concurrency);
+        const elapsedMs = Date.now() - started;
+        const failed = results.filter((result) => !result.ok);
+        const passed = results.filter((result) => result.ok);
+        const calledTools = new Set(openDatasetToolCallLog.map((entry) => entry.tool));
+        const requiredTools = [
+            "run_recon",
+            "log_recon_finding",
+            "findTools",
+            "getMethodology",
+            "start_eval_run",
+            "record_eval_result",
+            "complete_eval_run",
+            "run_closed_loop",
+            "run_mandatory_flywheel",
+            "search_all_knowledge",
+        ];
+        console.log(`[open-dataset-toolbench] dataset=${fixture.dataset} split=${fixture.split} tasks=${taskLimit} concurrency=${concurrency} pass=${passed.length}/${results.length} elapsedMs=${elapsedMs} toolCalls=${openDatasetToolCallLog.length}`);
+        if (failed.length > 0) {
+            console.error("[open-dataset-toolbench] failures:", failed.map((result) => ({
+                taskId: result.taskId,
+                workerIndex: result.workerIndex,
+                error: result.error,
+            })));
+        }
+        expect(failed.length).toBe(0);
+        expect(passed.length).toBe(taskLimit);
+        for (const requiredTool of requiredTools) {
+            expect(calledTools.has(requiredTool)).toBe(true);
+        }
+    });
+});
+//# sourceMappingURL=openDatasetParallelEvalToolbench.test.js.map

package/dist/__tests__/openDatasetParallelEvalToolbench.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openDatasetParallelEvalToolbench.test.js","sourceRoot":"","sources":["../../src/__tests__/openDatasetParallelEvalToolbench.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,cAAc,MAAM,8CAA8C,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAyCxD,MAAM,OAAO,GAAG,cAAgC,CAAC;AAEjD,MAAM,WAAW,GAAc;IAC7B,GAAG,iBAAiB;IACpB,GAAG,SAAS;IACZ,GAAG,gBAAgB;IACnB,GAAG,aAAa;IAChB,GAAG,aAAa;IAChB,GAAG,UAAU;IACb,GAAG,kBAAkB;IACrB,GAAG,mBAAmB;CACvB,CAAC;AACF,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;AAEnE,MAAM,sBAAsB,GAKvB,EAAE,CAAC;AAER,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,IAAY,EACZ,IAA6B,EAC7B,MAAc,EACd,KAAa;IAEb,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAiB,EAAE,WAAmB;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;IAEpF,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAC3B,WAAW,EACX;QACE,MAAM,EAAE,6BAA6B,IAAI,CAAC,EAAE,EAAE;QAC9C,WAAW,EAAE,uCAAuC,IAAI,CAAC,QAAQ,yBAAyB;QAC1F,cAAc,EAAE;YACd,SAAS,EAAE,yBAAyB;YACpC,YAAY,EAAE,4DAA4D;SAC3E;KACF,EACD,IAAI,CAAC,EAAE,EACP,aAAa,CACd,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,mBAAmB,EACnB;QACE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,QAAQ,EAAE,SAAS;QACnB,OAAO,EAAE,2BAA2B,IAAI,CAAC,EAAE,SAAS,IAAI,CAAC,QAAQ,wBAAwB,IAAI,CAAC,eAAe,GAAG;QAChH,SAAS,EAAE,aAAa;QACxB,SAAS,EAAE,kEAAkE;QAC7E,WAAW,EAAE,oEAAoE;KAClF,EACD,IAAI,CAAC,EAAE,EACP,WAAW,CACZ,CAAC;IAEF,IAAI,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC9B,WAAW,EACX;QACE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAChC,QAAQ,EAAE,cAAc;KACzB,EACD,IAAI,CAAC,EAAE,EACP,YAAY,CACb,CAAQ,CAAC;IAEV,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvE,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC1B,WAAW,EACX,EAAE,KAAK,EAAE,4CAA4C,EAAE,QAAQ,EAAE,WAAW,EAAE,EAC9E,IAAI,CAAC,EAAE,EACP,qBAAqB,CACtB,CAAQ,CAAC;IACZ,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,aAAa,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,gBAAgB,EAChB,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAC3B,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;IACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAC7B,gBAAgB,EAChB;QACE,IAAI,EAAE,0BAA0B,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;QACvD,WAAW,EAAE,kCAAkC,IAAI,CAAC,QAAQ,iBAAiB,WAAW,GAAG;QAC3F,KAAK,EAAE;YACL;gBACE,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,MAAM,EAAE,kDAAkD,IAAI,CAAC,EAAE,EAAE;gBACnE,QAAQ,EACN,mGAAmG;aACtG;SACF;KACF,EACD,IAAI,CAAC,EAAE,EACP,gBAAgB,CACjB,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,oBAAoB,EACpB;QACE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,cAAc,UAAU,CAAC,KAAK,CAAC,MAAM,0CAA0C;QACvF,SAAS,EAAE;YACT,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,WAAW;YACX,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC;KACF,EACD,IAAI,CAAC,EAAE,EACP,oBAAoB,CACrB,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,mBAAmB,EACnB,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EACxB,IAAI,CAAC,EAAE,EACP,mBAAmB,CACpB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAChC,iBAAiB,EACjB;QACE,KAAK,EAAE;YACL,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,sBAAsB,IAAI,CAAC,EAAE,EAAE,EAAE;YAC1E,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE;YACpE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,IAAI,CAAC,EAAE,EAAE,EAAE;SACnF;KACF,EACD,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,CAC9B,wBAAwB,EACxB;QACE,MAAM,EAAE,8BAA8B,IAAI,CAAC,EAAE,EAAE;QAC/C,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;YACrF,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,oCAAoC,EAAE;YAC3F,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,qCAAqC,EAAE;YAC9F,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,EAAE;YACrF,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kCAAkC,EAAE;YAC1F,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;SAC1F;KACF,EACD,IAAI,CAAC,EAAE,EACP,wBAAwB,CACzB,CAAQ,CAAC;IACV,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,CAAC,MAAM,QAAQ,CAC/B,sBAAsB,EACtB,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,EAAE,EACpC,IAAI,CAAC,EAAE,EACP,sBAAsB,CACvB,CAAQ,CAAC;IACV,MAAM,CAAC,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAElD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,WAAW;QACX,EAAE,EAAE,IAAI;QACR,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAC/B,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;QACxC,aAAa,EAAE,SAAS,CAAC,YAAY;KACtC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,KAAoB,EAAE,WAAmB;IACpE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAC5E,CAAC,KAAK,IAAI,EAAE;QACV,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;YAC9B,IAAI,SAAS,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAEtC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC;gBACH,OAAO,CAAC,SAAS,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,SAAS,CAAC,GAAG;oBACnB,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,WAAW;oBACX,EAAE,EAAE,KAAK;oBACT,SAAS,EAAE,CAAC;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,CAAC;oBAChB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,EAAE,CACL,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,QAAQ,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACzF,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEhD,MAAM,kBAAkB,GAAG,MAAM,CAAC,QAAQ,CACxC,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,GAAG,EACjD,EAAE,CACH,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7F,CAAC;QAEF,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAC1C,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,GAAG,EAClD,EAAE,CACH,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;QAEF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/E,MAAM,aAAa,GAAG;YACpB,WAAW;YACX,mBAAmB;YACnB,WAAW;YACX,gBAAgB;YAChB,gBAAgB;YAChB,oBAAoB;YACpB,mBAAmB;YACnB,iBAAiB;YACjB,wBAAwB;YACxB,sBAAsB;SACvB,CAAC;QAEF,OAAO,CAAC,GAAG,CACT,oCAAoC,OAAO,CAAC,OAAO,UAAU,OAAO,CAAC,KAAK,UAAU,SAAS,gBAAgB,WAAW,SAAS,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,cAAc,SAAS,cAAc,sBAAsB,CAAC,MAAM,EAAE,CACrO,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CACX,oCAAoC,EACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/tools.test.js

@@ -17,6 +17,7 @@ import { webTools } from "../tools/webTools.js";
 import { githubTools } from "../tools/githubTools.js";
 import { documentationTools } from "../tools/documentationTools.js";
 import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
+import { selfEvalTools } from "../tools/selfEvalTools.js";
 // Assemble all tools like index.ts does
 const domainTools = [
     ...verificationTools,
@@ -31,14 +32,15 @@ const domainTools = [
     ...githubTools,
     ...documentationTools,
     ...agentBootstrapTools,
+    ...selfEvalTools,
 ];
 const allTools = [...domainTools, ...createMetaTools(domainTools)];
 // ═══════════════════════════════════════════════════════════════════════════
 // STATIC LAYER — structure validation
 // ═══════════════════════════════════════════════════════════════════════════
 describe("Static: tool structure", () => {
-    it("should have 51 tools total", () => {
-        expect(allTools.length).toBe(51);
+    it("should have 60 tools total", () => {
+        expect(allTools.length).toBe(60);
     });
     it("every tool has name, description, inputSchema, handler", () => {
         for (const tool of allTools) {
@@ -248,7 +250,8 @@ describe("Static: new methodology topics", () => {
         expect(topics).toContain("telemetry_setup");
         expect(topics).toContain("agents_md_maintenance");
         expect(topics).toContain("agent_bootstrap");
-        expect(topics.length).toBe(15); // All topics listed in overview
+        expect(topics).toContain("autonomous_maintenance");
+        expect(topics.length).toBe(17); // All topics listed in overview
     });
 });
 describe("Unit: setup_local_env", () => {
@@ -276,6 +279,160 @@ describe("Unit: research_job_market", () => {
         expect(result.commonRequirements.length).toBeGreaterThan(0);
     });
 });
+describe("Static: autonomous maintenance tools", () => {
+    it("should include all autonomous tools", () => {
+        const names = allTools.map((t) => t.name);
+        expect(names).toContain("assess_risk");
+        expect(names).toContain("decide_re_update");
+        expect(names).toContain("run_self_maintenance");
+        expect(names).toContain("scaffold_directory");
+        expect(names).toContain("run_autonomous_loop");
+    });
+    it("assess_risk requires action parameter", () => {
+        const tool = allTools.find((t) => t.name === "assess_risk");
+        expect(tool.inputSchema.required).toContain("action");
+    });
+    it("decide_re_update requires targetContent and contentType", () => {
+        const tool = allTools.find((t) => t.name === "decide_re_update");
+        expect(tool.inputSchema.required).toContain("targetContent");
+        expect(tool.inputSchema.required).toContain("contentType");
+    });
+    it("scaffold_directory requires component", () => {
+        const tool = allTools.find((t) => t.name === "scaffold_directory");
+        expect(tool.inputSchema.required).toContain("component");
+    });
+    it("run_autonomous_loop requires goal", () => {
+        const tool = allTools.find((t) => t.name === "run_autonomous_loop");
+        expect(tool.inputSchema.required).toContain("goal");
+    });
+});
+describe("Unit: assess_risk", () => {
+    it("should classify known high-risk actions", async () => {
+        const tool = allTools.find((t) => t.name === "assess_risk");
+        const result = (await tool.handler({ action: "push_to_remote" }));
+        expect(result.assessment.tier).toBe("high");
+        expect(result.assessment.recommendation).toBe("require_confirmation");
+    });
+    it("should classify known low-risk actions", async () => {
+        const tool = allTools.find((t) => t.name === "assess_risk");
+        const result = (await tool.handler({ action: "read_file" }));
+        expect(result.assessment.tier).toBe("low");
+        expect(result.assessment.recommendation).toBe("auto_approve");
+    });
+    it("should use heuristics for unknown actions", async () => {
+        const tool = allTools.find((t) => t.name === "assess_risk");
+        const result = (await tool.handler({ action: "delete everything" }));
+        expect(result.assessment.tier).toBe("high");
+        expect(result.reasoning).toContain("Heuristic");
+    });
+});
+describe("Unit: decide_re_update", () => {
+    it("should recommend update_existing for instruction files", async () => {
+        const tool = allTools.find((t) => t.name === "decide_re_update");
+        const result = (await tool.handler({
+            targetContent: "New agent instructions",
+            contentType: "instructions",
+            existingFiles: ["AGENTS.md", "README.md"],
+        }));
+        expect(result.action).toBe("update_existing");
+        expect(result.existingFile).toBe("AGENTS.md");
+    });
+    it("should recommend create_new when no matching files exist", async () => {
+        const tool = allTools.find((t) => t.name === "decide_re_update");
+        const result = (await tool.handler({
+            targetContent: "Some random config",
+            contentType: "config",
+            existingFiles: [],
+        }));
+        expect(result.action).toBe("create_new");
+    });
+});
+describe("Unit: run_self_maintenance", () => {
+    it("should return maintenance report with quick scope", async () => {
+        const tool = allTools.find((t) => t.name === "run_self_maintenance");
+        const result = (await tool.handler({ scope: "quick" }));
+        expect(result).toHaveProperty("checksPerformed");
+        expect(result).toHaveProperty("issuesFound");
+        expect(result).toHaveProperty("actionsExecuted");
+        expect(result).toHaveProperty("updatesRecommended");
+        expect(result).toHaveProperty("nextScheduledCheck");
+        expect(result.checksPerformed.length).toBeGreaterThan(0);
+    });
+});
+describe("Unit: scaffold_directory", () => {
+    it("should return scaffold structure for agent_loop", async () => {
+        const tool = allTools.find((t) => t.name === "scaffold_directory");
+        const result = (await tool.handler({ component: "agent_loop" }));
+        expect(result.component).toBe("agent_loop");
+        expect(result.structure.files.length).toBeGreaterThan(0);
+        expect(result.createCommands.length).toBeGreaterThan(0);
+        expect(result.nextSteps.length).toBeGreaterThan(0);
+    });
+    it("should throw for unknown component", async () => {
+        const tool = allTools.find((t) => t.name === "scaffold_directory");
+        await expect(tool.handler({ component: "unknown_component" })).rejects.toThrow("Unknown component");
+    });
+});
+describe("Unit: run_autonomous_loop", () => {
+    it("should complete loop with goal", async () => {
+        const tool = allTools.find((t) => t.name === "run_autonomous_loop");
+        const result = (await tool.handler({
+            goal: "Test autonomous verification",
+            maxIterations: 3,
+            maxDurationMs: 5000,
+        }));
+        expect(result.goal).toBe("Test autonomous verification");
+        expect(result.iterations).toBeGreaterThan(0);
+        expect(result.iterations).toBeLessThanOrEqual(3);
+        expect(["completed", "stopped", "timeout", "failed"]).toContain(result.status);
+        expect(result.results.length).toBeGreaterThan(0);
+    });
+});
+describe("Static: autonomous_maintenance methodology", () => {
+    it("should return autonomous_maintenance methodology with 5 steps", async () => {
+        const tool = allTools.find((t) => t.name === "getMethodology");
+        const result = (await tool.handler({ topic: "autonomous_maintenance" }));
+        expect(result.title).toContain("Autonomous Self-Maintenance");
+        expect(result.steps.length).toBe(5);
+        expect(result.steps[0].name).toBe("Assess Risk Before Action");
+        expect(result.steps[1].name).toBe("Re-Update Before Create");
+        expect(result).toHaveProperty("riskTiers");
+        expect(result).toHaveProperty("patterns");
+    });
+});
+describe("Static: self-eval tools", () => {
+    it("should include all 4 self-eval tools", () => {
+        const names = allTools.map((t) => t.name);
+        expect(names).toContain("log_tool_call");
+        expect(names).toContain("get_trajectory_analysis");
+        expect(names).toContain("get_self_eval_report");
+        expect(names).toContain("get_improvement_recommendations");
+    });
+    it("log_tool_call requires sessionId and toolName", () => {
+        const tool = allTools.find((t) => t.name === "log_tool_call");
+        expect(tool.inputSchema.required).toContain("sessionId");
+        expect(tool.inputSchema.required).toContain("toolName");
+    });
+    it("get_improvement_recommendations has focus enum", () => {
+        const tool = allTools.find((t) => t.name === "get_improvement_recommendations");
+        const focusProp = tool.inputSchema.properties.focus;
+        expect(focusProp.enum).toContain("tools");
+        expect(focusProp.enum).toContain("process");
+        expect(focusProp.enum).toContain("quality");
+        expect(focusProp.enum).toContain("knowledge");
+        expect(focusProp.enum).toContain("all");
+    });
+});
+describe("Static: self_reinforced_learning methodology", () => {
+    it("should return self_reinforced_learning methodology with 5 steps", async () => {
+        const tool = allTools.find((t) => t.name === "getMethodology");
+        const result = (await tool.handler({ topic: "self_reinforced_learning" }));
+        expect(result.title).toContain("Self-Reinforced Learning");
+        expect(result.steps.length).toBe(5);
+        expect(result.steps[0].name).toBe("Instrument");
+        expect(result.steps[4].name).toBe("Apply & Re-Analyze");
+    });
+});
 // ═══════════════════════════════════════════════════════════════════════════
 // UNIT LAYER — individual tool behavior
 // ═══════════════════════════════════════════════════════════════════════════
@@ -373,6 +530,98 @@ describe("Integration: full verification cycle chain", () => {
         expect(abandoned.abandoned).toBe(true);
     });
 });
+describe("Unit: log_tool_call", () => {
+    it("should log a tool call and return confirmation", async () => {
+        const tool = findTool("log_tool_call");
+        const result = (await tool.handler({
+            sessionId: "test-session-001",
+            toolName: "run_recon",
+            durationMs: 42,
+            resultStatus: "success",
+            phase: "recon",
+        }));
+        expect(result.logged).toBe(true);
+        expect(result.sessionId).toBe("test-session-001");
+        expect(result.toolName).toBe("run_recon");
+        expect(result.resultStatus).toBe("success");
+    });
+    it("should log error tool calls", async () => {
+        const tool = findTool("log_tool_call");
+        const result = (await tool.handler({
+            sessionId: "test-session-001",
+            toolName: "web_search",
+            durationMs: 1500,
+            resultStatus: "error",
+            error: "API key not configured",
+            phase: "recon",
+        }));
+        expect(result.logged).toBe(true);
+        expect(result.resultStatus).toBe("error");
+    });
+});
+describe("Unit: get_trajectory_analysis", () => {
+    it("should return trajectory analysis with logged data", async () => {
+        // Log a few calls first
+        const logTool = findTool("log_tool_call");
+        await logTool.handler({ sessionId: "traj-test", toolName: "findTools", durationMs: 10, phase: "meta" });
+        await logTool.handler({ sessionId: "traj-test", toolName: "run_recon", durationMs: 20, phase: "recon" });
+        await logTool.handler({ sessionId: "traj-test", toolName: "log_recon_finding", durationMs: 15, phase: "recon" });
+        const tool = findTool("get_trajectory_analysis");
+        const result = (await tool.handler({ sessionId: "traj-test" }));
+        expect(result.totalCalls).toBeGreaterThanOrEqual(3);
+        expect(result.uniqueTools).toBeGreaterThanOrEqual(3);
+        expect(result.topTools.length).toBeGreaterThan(0);
+    });
+    it("should return empty message when no data exists for session", async () => {
+        const tool = findTool("get_trajectory_analysis");
+        const result = (await tool.handler({ sessionId: "nonexistent-session-xyz" }));
+        expect(result.totalCalls).toBe(0);
+        expect(result.message).toBeTruthy();
+    });
+});
+describe("Unit: get_self_eval_report", () => {
+    it("should return health report with all sections", async () => {
+        const tool = findTool("get_self_eval_report");
+        const result = (await tool.handler({ sinceDaysAgo: 30 }));
+        expect(result).toHaveProperty("healthScore");
+        expect(result).toHaveProperty("healthGrade");
+        expect(result).toHaveProperty("verification");
+        expect(result).toHaveProperty("gaps");
+        expect(result).toHaveProperty("evalRuns");
+        expect(result).toHaveProperty("qualityGates");
+        expect(result).toHaveProperty("knowledge");
+        expect(result).toHaveProperty("toolTrajectory");
+        expect(typeof result.healthScore).toBe("number");
+        expect(["A", "B", "C", "D", "F"]).toContain(result.healthGrade);
+    });
+    it("should include details when requested", async () => {
+        const tool = findTool("get_self_eval_report");
+        const result = (await tool.handler({ sinceDaysAgo: 30, includeDetails: true }));
+        expect(result).toHaveProperty("cycleDetails");
+        expect(result).toHaveProperty("openGapDetails");
+    });
+});
+describe("Unit: get_improvement_recommendations", () => {
+    it("should return structured recommendations", async () => {
+        const tool = findTool("get_improvement_recommendations");
+        const result = (await tool.handler({ sinceDaysAgo: 30 }));
+        expect(typeof result.totalRecommendations).toBe("number");
+        expect(typeof result.highPriority).toBe("number");
+        expect(typeof result.mediumPriority).toBe("number");
+        expect(typeof result.lowPriority).toBe("number");
+        expect(Array.isArray(result.recommendations)).toBe(true);
+        expect(result).toHaveProperty("_selfReinforcement");
+        expect(result._selfReinforcement.nextSteps.length).toBe(4);
+    });
+    it("should filter by focus area", async () => {
+        const tool = findTool("get_improvement_recommendations");
+        const result = (await tool.handler({ sinceDaysAgo: 30, focus: "quality" }));
+        expect(result.focus).toBe("quality");
+        for (const rec of result.recommendations) {
+            expect(rec.category).toBe("quality");
+        }
+    });
+});
 describe("Integration: search finds logged gaps", () => {
     it("should find gaps via search_all_knowledge after logging", async () => {
         const uniqueMarker = `vitest-marker-${Date.now()}`;