cuonztech-agent-harness 0.5.0 → 0.5.2

package/dist/proxy/interceptor.js

@@ -6,6 +6,7 @@
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { VERSION } from "../version.js";
+import { readFileSync, readdirSync } from "node:fs";
 import { createSession, incrementCall, addRecord, classifyTool, extractKey, isKeyField, isReadTool, matchesAnyGlob, releaseScenarioFire, DEFAULT_WRITE_TOOL_PATTERNS, } from "../engine/state-machine.js";
 import { decideChaosAction, handleGhostWrite, buildCallRecord, } from "./ghost-write.js";
 function sleep(ms) {
@@ -36,7 +37,12 @@ export function classifyUpstreamTool(tool, config) {
         return { name, class: "write", source: "annotation destructiveHint=true" };
     }
     // Explicit --write-tools replaces the default pattern list (documented).
-    if (explicitWrite.length === 0 && matchesAnyGlob(name, DEFAULT_WRITE_TOOL_PATTERNS)) {
+    // The defaults are snake_case ("create_*"); a camelCase name is matched
+    // through its snake_case form, so "createAppointment" is a write too.
+    const snake = name.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
+    if (explicitWrite.length === 0 &&
+        (matchesAnyGlob(name, DEFAULT_WRITE_TOOL_PATTERNS) ||
+            matchesAnyGlob(snake, DEFAULT_WRITE_TOOL_PATTERNS))) {
         return { name, class: "write", source: "default write pattern" };
     }
     if (isReadTool(name)) {
@@ -46,6 +52,61 @@ export function classifyUpstreamTool(tool, config) {
 }
 const STDERR_TAIL_CHARS = 2000;
 const MAX_TOOL_LIST_PAGES = 1000;
+// Linux only (/proc). [ppid, starttime] of a process, or null if it is gone.
+function procStat(pid) {
+    try {
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf-8");
+        // The command name (field 2) may contain spaces and parentheses.
+        const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+        return { ppid: Number(fields[1]), startTime: fields[19] };
+    }
+    catch {
+        return null;
+    }
+}
+function listDescendants(root) {
+    if (root === null || process.platform !== "linux")
+        return [];
+    try {
+        const children = new Map();
+        for (const entry of readdirSync("/proc")) {
+            if (!/^\d+$/.test(entry))
+                continue;
+            const pid = Number(entry);
+            const stat = procStat(pid);
+            if (!stat)
+                continue;
+            const list = children.get(stat.ppid) ?? [];
+            list.push({ pid, startTime: stat.startTime });
+            children.set(stat.ppid, list);
+        }
+        const found = [];
+        const queue = [root];
+        while (queue.length > 0 && found.length < 1000) {
+            for (const child of children.get(queue.shift()) ?? []) {
+                found.push(child);
+                queue.push(child.pid);
+            }
+        }
+        return found;
+    }
+    catch {
+        return [];
+    }
+}
+function killSurvivors(processes) {
+    for (const ref of processes) {
+        // Same pid AND same start time: never a process that reused the number.
+        if (procStat(ref.pid)?.startTime !== ref.startTime)
+            continue;
+        try {
+            process.kill(ref.pid, "SIGKILL");
+        }
+        catch {
+            // already gone
+        }
+    }
+}
 export class ProxyInterceptor {
     upstreamClient = null;
     upstreamTransport = null;
@@ -56,7 +117,9 @@ export class ProxyInterceptor {
     upstreamAlive = false;
     closing = false;
     stderrTail = "";
-    // Called after every intercepted call has been recorded (--report-file).
+    // Called when a call is forwarded upstream and again once it has been
+    // recorded (--report-file). The answer to the agent waits for the second
+    // one, so the file is never behind what the agent has already seen.
     onCallRecorded = null;
     // Called once if the upstream connection drops while the proxy is running.
     onUpstreamLost = null;
@@ -99,7 +162,27 @@ export class ProxyInterceptor {
                 this.onUpstreamLost?.(this.stderrTail);
             }
         };
-        await this.upstreamClient.connect(this.upstreamTransport);
+        // connect() spawns the process before its first await.
+        const connecting = this.upstreamClient.connect(this.upstreamTransport);
+        const pid = this.upstreamTransport.pid;
+        try {
+            await connecting;
+        }
+        catch (err) {
+            // The SDK starts closing the transport on its own but does not wait
+            // for it; this process is about to exit, which left an upstream that
+            // never answered (a hanging start) running. It is our own child and
+            // has not been collected yet, so the pid is certainly still its own.
+            if (pid !== null) {
+                try {
+                    process.kill(pid, "SIGKILL");
+                }
+                catch {
+                    // already gone
+                }
+            }
+            throw err;
+        }
         this.upstreamAlive = true;
         // Discover upstream tools (full definitions, including inputSchema, so
         // the gateway can advertise them to its own client unchanged). The list
@@ -117,6 +200,7 @@ export class ProxyInterceptor {
         this.upstreamTools = discovered;
         this.session.knownToolsOnly = true;
         this.session.kind = "proxy";
+        this.session.pending = new Map();
         this.toolClassification = this.upstreamTools.map((tool) => classifyUpstreamTool(tool, this.config));
         for (const entry of this.toolClassification) {
             this.session.toolClassOverrides.set(entry.name, entry.class);
@@ -130,10 +214,18 @@ export class ProxyInterceptor {
     }
     async disconnectUpstream() {
         this.closing = true;
+        // A wrapper command ("sh -c ...", "npx ...") ends on its own, but the
+        // server it started may not: note its descendants first, and end those
+        // that are still there afterwards.
+        const descendants = listDescendants(this.upstreamTransport?.pid ?? null);
         if (this.upstreamClient) {
-            await this.upstreamClient.close();
+            await this.upstreamClient.close().catch(() => undefined);
             this.upstreamClient = null;
         }
+        // After a failed start the client never took over the transport.
+        await this.upstreamTransport?.close().catch(() => undefined);
+        this.upstreamTransport = null;
+        killSurvivors(descendants);
     }
     getUpstreamTools() {
         return this.upstreamTools;
@@ -156,6 +248,16 @@ export class ProxyInterceptor {
         return [...this.session.keyCapableTools];
     }
     async interceptToolCall(toolName, args) {
+        const result = await this.handleToolCall(toolName, args);
+        try {
+            await this.onCallRecorded?.();
+        }
+        catch {
+            // Reporting must never change what the agent gets back.
+        }
+        return result;
+    }
+    async handleToolCall(toolName, args) {
         // Everything up to the first await runs synchronously, so two calls
         // dispatched in parallel still get distinct, ordered write positions —
         // only one of them can be "the first write" a scenario fires on.
@@ -163,16 +265,10 @@ export class ProxyInterceptor {
         const callNumber = incrementCall(this.session.sessionId);
         const mergedArgs = { ...args };
         const key = extractKey(mergedArgs, this.session.keyFields);
-        // A tool the upstream executes but did not list: an explicit
-        // --write-tools/--read-tools pattern still applies to it.
-        if (!this.session.toolClassOverrides.has(toolName)) {
-            if (matchesAnyGlob(toolName, this.config.writeToolPatterns ?? [])) {
-                this.session.toolClassOverrides.set(toolName, "write");
-            }
-            else if (matchesAnyGlob(toolName, this.config.readToolPatterns ?? [])) {
-                this.session.toolClassOverrides.set(toolName, "read");
-            }
-        }
+        // A tool the upstream did not list is unclassified whatever its name
+        // and whatever --write-tools says: F2/F3/F5 answer without asking the
+        // upstream, so a call to a tool that does not exist would otherwise use
+        // up the scenario and the real write would never see a failure.
         const toolClass = classifyTool(this.session, toolName);
         // A dead upstream is reported as exactly that. No scenario is applied:
         // an injected "timeout" on top would hide the real problem.
@@ -197,6 +293,29 @@ export class ProxyInterceptor {
         let upstreamExecuted = false;
         let upstreamFailure = null;
         if (decision.executeUpstream) {
+            // Until the upstream answers, this call is "sent, effect unknown". A
+            // report written in the meantime — the session ends, the proxy is
+            // killed — must not leave it out: 0.5.0 gave PASS for a duplicate
+            // whose second write was still open when the client disconnected.
+            this.session.pending?.set(callNumber, {
+                ...buildCallRecord(callNumber, toolName, mergedArgs, null, "", false, {
+                    receivedAt,
+                    outcome: "upstream_error",
+                    toolClass,
+                    tookEffect: false,
+                    effectUnknown: true,
+                }),
+                timestamp: receivedAt,
+            });
+            // Waited for: if the proxy is killed the moment the upstream executes
+            // this call, the report file must already show it as open. (Not
+            // waiting left a 1-2 ms window in which the file still read PASS.)
+            try {
+                await this.onCallRecorded?.();
+            }
+            catch {
+                // see interceptToolCall
+            }
             try {
                 const result = (await this.upstreamClient.callTool({
                     name: toolName,
@@ -218,7 +337,9 @@ export class ProxyInterceptor {
                 }
             }
             catch (err) {
-                upstreamFailure = err instanceof Error ? err.message : String(err);
+                // An upstream that throws an McpError gets its "MCP error <code>: "
+                // prefix added a second time by the SDK client.
+                upstreamFailure = (err instanceof Error ? err.message : String(err)).replace(/^(MCP error -?\d+: )\1+/, "$1");
             }
         }
         // The call to the real upstream itself failed without an answer
@@ -305,7 +426,10 @@ export class ProxyInterceptor {
                 tookEffect: toolClass === "write" && !upstreamIsError,
                 scenarioTriggered: decision.scenarioTriggered && !ghostRejected,
                 upstreamResponse: upstreamText ?? "",
-            }), { ...upstreamResult, content, isError: upstreamIsError });
+            }), 
+            // isError is passed on only if the upstream sent one: adding
+            // "isError: false" to a success would change the upstream's answer.
+            { ...upstreamResult, content });
         }
         // No scenario triggered and no upstream call was attempted — cannot
         // happen with the current decision logic; answered as an error rather
@@ -322,11 +446,11 @@ export class ProxyInterceptor {
         }), { content: [{ type: "text", text }], isError: true });
     }
     finish(record, result, ghostUpstreamResponse = null) {
+        this.session.pending?.delete(record.callNumber);
         addRecord(this.session.sessionId, record);
         if (ghostUpstreamResponse) {
             handleGhostWrite(this.session, record.args, ghostUpstreamResponse);
         }
-        this.onCallRecorded?.();
         return result;
     }
 }

package/dist/scenarios/f1-f5.js

@@ -59,7 +59,7 @@ export const F3_MALFORMED_JSON = {
 // F4: Duplicate Dispatch
 export const F4_DUPLICATE_DISPATCH = {
     id: "F4",
-    name: "Duplicate Dispatch (Ghost-Write)",
+    name: "Duplicate Dispatch (no injected failure)",
     description: "Es wird nichts injiziert: die ersten beiden Schreibversuche gehen unverändert durch. Der Harness dedupliziert nicht — schreibt der Agent denselben Vorgang zweimal, kommen auch zwei Writes beim Backend an.",
     triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls <= 1 && isWriteCall,
     // Only used by the non-proxy session/benchmark simulation (tools/handlers.ts),

package/dist/server/server.js

@@ -1,53 +1,106 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import { z } from "zod";
 import { VERSION } from "../version.js";
 import { StartSessionSchema, ExecuteCallSchema, GetReportSchema, GetScoreSchema, ListScenariosSchema, ResetSessionSchema, DeleteSessionSchema, } from "../tools/schemas.js";
 import { handleStartSession, handleExecuteCall, handleGetReport, handleGetScore, handleListScenarios, handleResetSession, handleDeleteSession, } from "../tools/handlers.js";
+// Keywords that strict tool-calling stacks (vLLM/XGrammar and others) reject
+// or cannot compile. They add nothing here: the arguments are validated by
+// the Zod schemas when the tool is called.
+const DROPPED_SCHEMA_KEYS = new Set(["$schema", "pattern", "format", "propertyNames"]);
+function plainSchema(value) {
+    if (Array.isArray(value))
+        return value.map(plainSchema);
+    if (value === null || typeof value !== "object")
+        return value;
+    const out = {};
+    for (const [key, child] of Object.entries(value)) {
+        if (DROPPED_SCHEMA_KEYS.has(key))
+            continue;
+        // "properties" maps argument names to schemas: an argument may itself
+        // be called "format" and must not be dropped.
+        if (key === "properties" && child !== null && typeof child === "object") {
+            out[key] = Object.fromEntries(Object.entries(child).map(([name, schema]) => [
+                name,
+                plainSchema(schema),
+            ]));
+            continue;
+        }
+        if (key === "additionalProperties" && child !== null && typeof child === "object" &&
+            Object.keys(child).length === 0) {
+            out[key] = true;
+            continue;
+        }
+        out[key] = plainSchema(child);
+    }
+    return out;
+}
 export function createServer() {
+    const listed = [];
     const server = new McpServer({
         name: "cuonztech-agent-harness",
         version: VERSION,
     });
+    // Passes a tool's config through unchanged and keeps it for the
+    // hand-built tools/list below.
+    const keep = (name, config) => {
+        listed.push({ name, ...config });
+        return config;
+    };
     // Tool: start_session
-    server.registerTool("start_session", {
+    server.registerTool("start_session", keep("start_session", {
         title: "Start Test Session",
         description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls. Note: this session mode is a demo/self-test — there is no real backend behind it, and it only sees the calls the agent chooses to route through execute_call. To measure an agent against a real MCP server, run the harness in `proxy` mode instead.",
         inputSchema: StartSessionSchema,
-    }, async (args) => handleStartSession(args));
+    }), async (args) => handleStartSession(args));
     // Tool: execute_call
-    server.registerTool("execute_call", {
+    server.registerTool("execute_call", keep("execute_call", {
         title: "Execute Tracked Call",
         description: "Executes a tool call within an active session against a simulated backend. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit. Reads return a generic ok without data.",
         inputSchema: ExecuteCallSchema,
-    }, async (args) => handleExecuteCall(args));
+    }), async (args) => handleExecuteCall(args));
     // Tool: get_report
-    server.registerTool("get_report", {
+    server.registerTool("get_report", keep("get_report", {
         title: "Get Audit Report",
         description: "Generates the audit report for a session. Returns markdown (chat-friendly) or JSON. Also persists JSON to .cuonztech/audit-report.json.",
         inputSchema: GetReportSchema,
-    }, async (args) => handleGetReport(args));
+    }), async (args) => handleGetReport(args));
     // Tool: get_score
-    server.registerTool("get_score", {
+    server.registerTool("get_score", keep("get_score", {
         title: "Get Resilience Score",
         description: "Scores THIS session's own audit report (0-100: Duplicate Safety 60%, Failure Handling 40%; honesty is not measured) and generates copy-pasteable system-prompt hardening patches for whatever violations the session actually produced. Returns n/a instead of a score when nothing was tested. Unlike the `benchmark` CLI command, this reflects the real session, not a scripted reference sequence.",
         inputSchema: GetScoreSchema,
-    }, async (args) => handleGetScore(args));
+    }), async (args) => handleGetScore(args));
     // Tool: list_scenarios
-    server.registerTool("list_scenarios", {
+    server.registerTool("list_scenarios", keep("list_scenarios", {
         title: "List Available Scenarios",
         description: "Returns all available deterministic failure scenarios (F1–F5) with descriptions and audit metrics.",
         inputSchema: ListScenariosSchema,
-    }, async () => handleListScenarios());
+    }), async () => handleListScenarios());
     // Tool: reset_session
-    server.registerTool("reset_session", {
+    server.registerTool("reset_session", keep("reset_session", {
         title: "Reset Session",
         description: "Resets a session's call counter and history without deleting it. Useful for re-running the same scenario.",
         inputSchema: ResetSessionSchema,
-    }, async (args) => handleResetSession(args));
+    }), async (args) => handleResetSession(args));
     // Tool: delete_session
-    server.registerTool("delete_session", {
+    server.registerTool("delete_session", keep("delete_session", {
         title: "Delete Session",
         description: "Permanently deletes a session and its state.",
         inputSchema: DeleteSessionSchema,
-    }, async (args) => handleDeleteSession(args));
+    }), async (args) => handleDeleteSession(args));
+    // The SDK's generated tools/list carries "$schema", a UUID "pattern" and
+    // "format" on every session_id, "propertyNames" and an "execution" block.
+    // An external test lost 69 of 100 runs to a tool-calling stack that could
+    // not compile those. This replaces the listing only; calls still go
+    // through the SDK's own handler and Zod validation.
+    server.server.setRequestHandler(ListToolsRequestSchema, async () => ({
+        tools: listed.map((tool) => ({
+            name: tool.name,
+            title: tool.title,
+            description: tool.description,
+            inputSchema: plainSchema(z.toJSONSchema(z.object(tool.inputSchema), { io: "input" })),
+        })),
+    }));
     return server;
 }

package/dist/tools/schemas.js

@@ -44,7 +44,7 @@ export const StartSessionSchema = {
         .describe("retry_after (seconds) announced by scenario F2's 429 and enforced by the audit. Default 60."),
 };
 export const ExecuteCallSchema = {
-    session_id: z.string().uuid().describe("Active session ID."),
+    session_id: z.string().uuid().describe("Active session ID (the UUID returned by start_session)."),
     tool_name: z.string().describe("Name of the tool the agent is calling."),
     arguments: z
         .record(z.string(), z.unknown())
@@ -55,7 +55,7 @@ export const ExecuteCallSchema = {
         .describe("Idempotency key for duplicate detection."),
 };
 export const GetReportSchema = {
-    session_id: z.string().uuid().describe("Session ID to audit."),
+    session_id: z.string().uuid().describe("Session ID to audit (the UUID returned by start_session)."),
     format: z
         .enum(["json", "markdown"])
         .optional()
@@ -63,7 +63,7 @@ export const GetReportSchema = {
         .describe("Output format: markdown (default) or json."),
 };
 export const GetScoreSchema = {
-    session_id: z.string().uuid().describe("Session ID to score."),
+    session_id: z.string().uuid().describe("Session ID to score (the UUID returned by start_session)."),
     format: z
         .enum(["json", "markdown"])
         .optional()
@@ -72,8 +72,8 @@ export const GetScoreSchema = {
 };
 export const ListScenariosSchema = {};
 export const ResetSessionSchema = {
-    session_id: z.string().uuid().describe("Session ID to reset."),
+    session_id: z.string().uuid().describe("Session ID to reset (the UUID returned by start_session)."),
 };
 export const DeleteSessionSchema = {
-    session_id: z.string().uuid().describe("Session ID to delete."),
+    session_id: z.string().uuid().describe("Session ID to delete (the UUID returned by start_session)."),
 };

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "cuonztech-agent-harness",
-  "version": "0.5.0",
+  "version": "0.5.2",
+  "mcpName": "io.github.cuonztech/agent-harness",
   "description": "Chaos-Proxy für MCP-Server: injiziert F1-F5-Fehlerszenarien (Ghost-Write nach Timeout, 429, defektes JSON, Duplicate Dispatch, 500) vor einem echten Upstream-MCP-Server und prüft, ob ein Agent Schreibvorgänge doppelt ausführt. Mit Resilience Score und System-Prompt-Patches für beobachtetes Fehlverhalten.",
   "type": "module",
   "main": "dist/index.js",
@@ -45,7 +46,8 @@
   "files": [
     "dist/**/*.js",
     "README.md",
-    "LICENSE"
+    "LICENSE",
+    "server.json"
   ],
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.32.1",

package/server.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.cuonztech/agent-harness",
+  "description": "MCP chaos-testing toolkit. Default: simulated demo. Configure proxy mode to test a real agent.",
+  "repository": {
+    "url": "https://github.com/cuonztech/agent-harness",
+    "source": "github"
+  },
+  "version": "0.5.2",
+  "packages": [
+    {
+      "registryType": "npm",
+      "identifier": "cuonztech-agent-harness",
+      "version": "0.5.2",
+      "transport": {
+        "type": "stdio"
+      }
+    }
+  ]
+}