cuonztech-agent-harness 0.4.0 → 0.5.1

package/dist/proxy/gateway.js

@@ -1,91 +1,97 @@
 // Builds the MCP server exposed in `proxy` CLI mode: a transparent gateway
 // that advertises the upstream server's real tools (unchanged, discovered
-// via ProxyInterceptor) plus a handful of the harness's own audit tools
-// scoped to the proxy's single session. Uses the low-level Server class
-// instead of McpServer because upstream tool schemas arrive as raw JSON
-// Schema (from the upstream's own `listTools()`), not Zod — McpServer's
-// `registerTool` only accepts a Zod raw shape.
+// via ProxyInterceptor) plus — unless switched off — two read-only audit
+// tools scoped to the proxy's single session. Uses the low-level Server
+// class instead of McpServer because upstream tool schemas arrive as raw
+// JSON Schema (from the upstream's own `listTools()`), not Zod.
+//
+// Only the `tools` capability is proxied. Resources, prompts and
+// notifications of the upstream are NOT forwarded (documented limitation).
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { ListToolsRequestSchema, CallToolRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
-import { z } from "zod";
 import { VERSION } from "../version.js";
-import { GetReportSchema, GetScoreSchema, ListScenariosSchema, ResetSessionSchema, DeleteSessionSchema, } from "../tools/schemas.js";
-import { handleGetReport, handleGetScore, handleListScenarios, handleResetSession, handleDeleteSession, } from "../tools/handlers.js";
-// session_id is required in the shared schemas (used by the multi-session
-// stdio server), but a proxy only ever has one session — its own. This
-// override makes it optional so the proxy's CallTool handler can default it
-// to `interceptor.sessionId` without touching the shared schemas.
-function optionalSessionId(schema) {
-    return { ...schema, session_id: schema.session_id.optional() };
-}
+import { handleGetReport, handleGetScore } from "../tools/handlers.js";
+// Namespaced so they cannot shadow an upstream tool that happens to be
+// called get_report/get_score. Deliberately read-only: the agent under test
+// shares this tool list, and a reset/delete tool would let it erase its own
+// violations. Hide them completely with --audit-tools off and read the
+// result from --report-file instead.
+export const AUDIT_TOOL_PREFIX = "cuonztech__";
+// Written out by hand instead of generated from the shared Zod schemas:
+// those mark `format` as required (it has a default) and add a session_id
+// with a UUID `pattern` that has no meaning in a single-session proxy.
+const FORMAT_INPUT_SCHEMA = {
+    type: "object",
+    properties: {
+        format: {
+            type: "string",
+            enum: ["markdown", "json"],
+            default: "markdown",
+            description: "Output format: markdown (default) or json.",
+        },
+    },
+    additionalProperties: false,
+};
 const OWN_TOOLS = {
-    get_report: {
+    [`${AUDIT_TOOL_PREFIX}get_report`]: {
         title: "Get Audit Report",
-        description: "Generates the audit report for this proxy session. Returns markdown (chat-friendly) or JSON.",
-        schema: optionalSessionId(GetReportSchema),
+        description: "CuonzTech Agent-Harness audit tool (not part of the upstream server): returns the audit report for this proxy session as markdown or JSON.",
     },
-    get_score: {
+    [`${AUDIT_TOOL_PREFIX}get_score`]: {
         title: "Get Resilience Score",
-        description: "Scores this proxy session's own audit report (0-100) and generates system-prompt hardening patches for whatever violations the connected agent actually produced.",
-        schema: optionalSessionId(GetScoreSchema),
-    },
-    list_scenarios: {
-        title: "List Available Scenarios",
-        description: "Returns all available deterministic failure scenarios (F1–F5) with descriptions and audit metrics.",
-        schema: ListScenariosSchema,
-    },
-    reset_session: {
-        title: "Reset Session",
-        description: "Resets this proxy session's call counter and history without deleting it.",
-        schema: optionalSessionId(ResetSessionSchema),
-    },
-    delete_session: {
-        title: "Delete Session",
-        description: "Permanently deletes this proxy session's state.",
-        schema: optionalSessionId(DeleteSessionSchema),
+        description: "CuonzTech Agent-Harness audit tool (not part of the upstream server): scores this proxy session (0-100, or n/a when nothing was tested) and lists system-prompt hardening patches for the violations actually observed.",
     },
 };
 function isOwnTool(name) {
-    return name in OWN_TOOLS;
+    return Object.prototype.hasOwnProperty.call(OWN_TOOLS, name);
 }
-export function createProxyServer(interceptor) {
-    const server = new Server({ name: "cuonztech-agent-harness-proxy", version: VERSION }, { capabilities: { tools: {} } });
+// Upstream tools whose name equals one of the harness's own audit tools.
+// The proxy refuses to start on a collision rather than silently answering
+// such calls itself.
+export function findToolNameCollisions(upstreamTools) {
+    return upstreamTools.map((t) => t.name).filter((name) => isOwnTool(name));
+}
+export function createProxyServer(interceptor, options = {}) {
+    const auditTools = options.auditTools ?? true;
+    const instructions = interceptor.getUpstreamInstructions();
+    const server = new Server({ name: "cuonztech-agent-harness-proxy", version: VERSION }, { capabilities: { tools: {} }, ...(instructions ? { instructions } : {}) });
     server.setRequestHandler(ListToolsRequestSchema, async () => {
         const upstreamDefs = interceptor.getUpstreamTools();
-        const ownDefs = Object.keys(OWN_TOOLS).map((name) => {
-            const tool = OWN_TOOLS[name];
-            return {
-                name,
-                title: tool.title,
-                description: tool.description,
-                inputSchema: z.toJSONSchema(z.object(tool.schema)),
-            };
-        });
+        if (!auditTools)
+            return { tools: upstreamDefs };
+        const ownDefs = Object.keys(OWN_TOOLS).map((name) => ({
+            name,
+            title: OWN_TOOLS[name].title,
+            description: OWN_TOOLS[name].description,
+            inputSchema: FORMAT_INPUT_SCHEMA,
+            annotations: { readOnlyHint: true },
+        }));
         return { tools: [...upstreamDefs, ...ownDefs] };
     });
     server.setRequestHandler(CallToolRequestSchema, async (request) => {
         const { name, arguments: rawArgs } = request.params;
         const toolArgs = (rawArgs ?? {});
-        if (isOwnTool(name)) {
-            const withSession = {
-                session_id: interceptor.sessionId,
-                ...toolArgs,
-            };
-            switch (name) {
-                case "get_report":
-                    return handleGetReport(withSession);
-                case "get_score":
-                    return handleGetScore(withSession);
-                case "list_scenarios":
-                    return handleListScenarios();
-                case "reset_session":
-                    return handleResetSession(withSession);
-                case "delete_session":
-                    return handleDeleteSession(withSession);
+        if (auditTools && isOwnTool(name)) {
+            const rawFormat = toolArgs["format"] ?? "markdown";
+            const format = typeof rawFormat === "string" ? rawFormat.toLowerCase() : rawFormat;
+            if (format !== "markdown" && format !== "json") {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Error: format must be "markdown" or "json".`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+            const withSession = { session_id: interceptor.sessionId, format };
+            if (name === `${AUDIT_TOOL_PREFIX}get_report`) {
+                return handleGetReport(withSession, { persist: false });
             }
+            return handleGetScore(withSession);
         }
         return interceptor.interceptToolCall(name, toolArgs);
     });
     return server;
 }
-//# sourceMappingURL=gateway.js.map

package/dist/proxy/ghost-write.js

@@ -1,6 +1,24 @@
 import { getScenarioById } from "../scenarios/f1-f5.js";
-import { markGhostCommitted, isWriteTool } from "../engine/state-machine.js";
-export function decideChaosAction(session, callNumber, toolName, args) {
+import { markGhostCommitted, classifyTool, reserveScenarioFire, extractKey, } from "../engine/state-machine.js";
+// The response body a scenario hands to the agent. Only the 429 body depends
+// on the session: its retry_after is configurable (--retry-after /
+// retry_after_sec), because that is the value the audit later measures the
+// agent's next call against.
+export function resolveScenarioBody(scenario, session) {
+    const resp = scenario.simulatedResponse;
+    if (resp.type === "rate_limit" && typeof resp.body !== "string") {
+        return {
+            ...resp.body,
+            retry_after: session.retryAfterSec,
+            message: `Too many requests. Retry after ${session.retryAfterSec} seconds.`,
+        };
+    }
+    return resp.body;
+}
+// Must be called synchronously on receipt of a call (before any await): if
+// the scenario fires, that is reserved right here — see
+// reserveScenarioFire().
+export function decideChaosAction(session, callNumber, toolName, _args) {
     const noChaos = {
         injectError: false,
         errorType: null,
@@ -8,14 +26,17 @@ export function decideChaosAction(session, callNumber, toolName, args) {
         errorBody: null,
         executeUpstream: true,
         delayMs: 0,
+        scenarioTriggered: false,
+        retryAfterSec: null,
     };
-    const isWriteCall = isWriteTool(toolName, session.writeToolPatterns);
-    const priorWriteCalls = session.history.filter((r) => isWriteTool(r.toolName, session.writeToolPatterns)).length;
+    const isWriteCall = classifyTool(session, toolName) === "write";
+    const priorWriteCalls = session.scenarioFireCount;
     // Deterministic scenario mode
     if (session.scenarioId) {
         const scenario = getScenarioById(session.scenarioId);
         if (scenario &&
             scenario.triggerCondition({ callNumber, toolName, isWriteCall, priorWriteCalls })) {
+            reserveScenarioFire(session);
             const resp = scenario.simulatedResponse;
             const isGhostWrite = resp.type === "timeout" && isWriteCall;
             // F4 (duplicate dispatch) must also actually reach the upstream: it
@@ -30,9 +51,11 @@ export function decideChaosAction(session, callNumber, toolName, args) {
                 injectError: true,
                 errorType: resp.type === "success" ? null : `[${resp.type.toUpperCase()}]`,
                 statusCode: resp.statusCode ?? 500,
-                errorBody: resp.body,
+                errorBody: resolveScenarioBody(scenario, session),
                 executeUpstream,
                 delayMs: resp.delayMs ?? 0,
+                scenarioTriggered: true,
+                retryAfterSec: resp.type === "rate_limit" ? session.retryAfterSec : null,
             };
         }
         return noChaos;
@@ -43,24 +66,28 @@ export function decideChaosAction(session, callNumber, toolName, args) {
             injectError: true,
             errorType: "[CHAOS_INJECTED]",
             statusCode: 500,
+            // Worded like a real server error: a message naming the injection
+            // would tell the agent under test that it is being tested.
             errorBody: {
-                error: "CHAOS_ERROR",
-                message: "Stochastically injected failure.",
+                error: "INTERNAL_SERVER_ERROR",
+                message: "The server encountered an internal error.",
             },
             executeUpstream: false,
             delayMs: 0,
+            scenarioTriggered: false,
+            retryAfterSec: null,
         };
     }
     return noChaos;
 }
 export function handleGhostWrite(session, args, upstreamResponse) {
-    const key = args["idempotency_key"] ?? null;
+    const key = extractKey(args, session.keyFields) ?? null;
     if (key && upstreamResponse) {
         markGhostCommitted(session.sessionId, key);
     }
     return { upstreamResponse, key };
 }
-export function buildCallRecord(callNumber, toolName, args, injectedError, response, upstreamExecuted) {
+export function buildCallRecord(callNumber, toolName, args, injectedError, response, upstreamExecuted, extra = {}) {
     return {
         callNumber,
         toolName,
@@ -69,6 +96,6 @@ export function buildCallRecord(callNumber, toolName, args, injectedError, respo
         response,
         upstreamExecuted,
         timestamp: Date.now(),
+        ...extra,
     };
 }
-//# sourceMappingURL=ghost-write.js.map