cuonztech-agent-harness 0.4.0 → 0.5.0

package/dist/proxy/interceptor.js

@@ -6,20 +6,67 @@
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { VERSION } from "../version.js";
-import { createSession, incrementCall, addRecord, } from "../engine/state-machine.js";
+import { createSession, incrementCall, addRecord, classifyTool, extractKey, isKeyField, isReadTool, matchesAnyGlob, releaseScenarioFire, DEFAULT_WRITE_TOOL_PATTERNS, } from "../engine/state-machine.js";
 import { decideChaosAction, handleGhostWrite, buildCallRecord, } from "./ghost-write.js";
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+// Decides once, at startup, what each upstream tool is. Order of precedence:
+// explicit flags, then the tool's own MCP annotations, then name patterns.
+// Anything left over stays "unclassified" — and is reported as such instead
+// of being quietly treated as a harmless read.
+export function classifyUpstreamTool(tool, config) {
+    const name = tool.name;
+    const explicitWrite = config.writeToolPatterns ?? [];
+    const explicitRead = config.readToolPatterns ?? [];
+    if (matchesAnyGlob(name, explicitWrite)) {
+        return { name, class: "write", source: "--write-tools" };
+    }
+    if (matchesAnyGlob(name, explicitRead)) {
+        return { name, class: "read", source: "--read-tools" };
+    }
+    const annotations = tool.annotations;
+    if (annotations?.readOnlyHint === true) {
+        return { name, class: "read", source: "annotation readOnlyHint=true" };
+    }
+    if (annotations?.readOnlyHint === false) {
+        return { name, class: "write", source: "annotation readOnlyHint=false" };
+    }
+    if (annotations?.destructiveHint === true) {
+        return { name, class: "write", source: "annotation destructiveHint=true" };
+    }
+    // Explicit --write-tools replaces the default pattern list (documented).
+    if (explicitWrite.length === 0 && matchesAnyGlob(name, DEFAULT_WRITE_TOOL_PATTERNS)) {
+        return { name, class: "write", source: "default write pattern" };
+    }
+    if (isReadTool(name)) {
+        return { name, class: "read", source: "read prefix" };
+    }
+    return { name, class: "unclassified", source: "no annotation, no pattern match" };
+}
+const STDERR_TAIL_CHARS = 2000;
+const MAX_TOOL_LIST_PAGES = 1000;
 export class ProxyInterceptor {
     upstreamClient = null;
     upstreamTransport = null;
     session;
     config;
     upstreamTools = [];
+    toolClassification = [];
+    upstreamAlive = false;
+    closing = false;
+    stderrTail = "";
+    // Called after every intercepted call has been recorded (--report-file).
+    onCallRecorded = null;
+    // Called once if the upstream connection drops while the proxy is running.
+    onUpstreamLost = null;
     constructor(config) {
         this.config = config;
-        this.session = createSession(config.scenarioId ?? null, config.mode ?? "deterministic", config.errorRate ?? 0.0, config.writeToolPatterns);
+        this.session = createSession(config.scenarioId ?? null, config.mode ?? "deterministic", config.errorRate ?? 0.0, config.writeToolPatterns, {
+            readToolPatterns: config.readToolPatterns,
+            keyFields: config.keyFields,
+            retryAfterSec: config.retryAfterSec,
+        });
     }
     get sessionId() {
         return this.session.sessionId;
@@ -38,14 +85,51 @@ export class ProxyInterceptor {
             },
             stderr: "pipe",
         });
+        // Keep the last part of the upstream's stderr: if it dies, that is
+        // usually where the reason is. Reading the pipe also keeps a chatty
+        // upstream from blocking on a full buffer.
+        this.upstreamTransport.stderr?.on("data", (chunk) => {
+            this.stderrTail = (this.stderrTail + chunk.toString()).slice(-STDERR_TAIL_CHARS);
+        });
         this.upstreamClient = new Client({ name: "cuonztech-harness-proxy", version: VERSION }, { capabilities: {} });
+        this.upstreamClient.onclose = () => {
+            const wasAlive = this.upstreamAlive;
+            this.upstreamAlive = false;
+            if (wasAlive && !this.closing) {
+                this.onUpstreamLost?.(this.stderrTail);
+            }
+        };
         await this.upstreamClient.connect(this.upstreamTransport);
+        this.upstreamAlive = true;
         // Discover upstream tools (full definitions, including inputSchema, so
-        // the gateway can advertise them to its own client unchanged)
-        const tools = await this.upstreamClient.listTools();
-        this.upstreamTools = tools.tools;
+        // the gateway can advertise them to its own client unchanged). The list
+        // may be paginated: follow nextCursor to the end, or every tool after
+        // the first page would be invisible to the agent and unknown to the audit.
+        const discovered = [];
+        let cursor;
+        for (let page = 0; page < MAX_TOOL_LIST_PAGES; page++) {
+            const result = await this.upstreamClient.listTools(cursor ? { cursor } : undefined);
+            discovered.push(...result.tools);
+            cursor = result.nextCursor;
+            if (!cursor)
+                break;
+        }
+        this.upstreamTools = discovered;
+        this.session.knownToolsOnly = true;
+        this.session.kind = "proxy";
+        this.toolClassification = this.upstreamTools.map((tool) => classifyUpstreamTool(tool, this.config));
+        for (const entry of this.toolClassification) {
+            this.session.toolClassOverrides.set(entry.name, entry.class);
+        }
+        for (const tool of this.upstreamTools) {
+            const properties = (tool.inputSchema?.properties ?? {});
+            if (Object.keys(properties).some((p) => isKeyField(p, this.session.keyFields))) {
+                this.session.keyCapableTools.add(tool.name);
+            }
+        }
     }
     async disconnectUpstream() {
+        this.closing = true;
         if (this.upstreamClient) {
             await this.upstreamClient.close();
             this.upstreamClient = null;
@@ -54,120 +138,195 @@ export class ProxyInterceptor {
     getUpstreamTools() {
         return this.upstreamTools;
     }
+    getToolClassification() {
+        return this.toolClassification;
+    }
     getSession() {
         return this.session;
     }
+    getUpstreamStderrTail() {
+        return this.stderrTail;
+    }
+    // The upstream's own `instructions` from its initialize result, so the
+    // gateway can hand them on to the agent unchanged.
+    getUpstreamInstructions() {
+        return this.upstreamClient?.getInstructions();
+    }
+    getKeyCapableTools() {
+        return [...this.session.keyCapableTools];
+    }
     async interceptToolCall(toolName, args) {
+        // Everything up to the first await runs synchronously, so two calls
+        // dispatched in parallel still get distinct, ordered write positions —
+        // only one of them can be "the first write" a scenario fires on.
+        const receivedAt = Date.now();
         const callNumber = incrementCall(this.session.sessionId);
-        // Merge idempotency key into args
         const mergedArgs = { ...args };
-        const key = args["idempotency_key"];
-        // Decide chaos action
+        const key = extractKey(mergedArgs, this.session.keyFields);
+        // A tool the upstream executes but did not list: an explicit
+        // --write-tools/--read-tools pattern still applies to it.
+        if (!this.session.toolClassOverrides.has(toolName)) {
+            if (matchesAnyGlob(toolName, this.config.writeToolPatterns ?? [])) {
+                this.session.toolClassOverrides.set(toolName, "write");
+            }
+            else if (matchesAnyGlob(toolName, this.config.readToolPatterns ?? [])) {
+                this.session.toolClassOverrides.set(toolName, "read");
+            }
+        }
+        const toolClass = classifyTool(this.session, toolName);
+        // A dead upstream is reported as exactly that. No scenario is applied:
+        // an injected "timeout" on top would hide the real problem.
+        if (!this.upstreamAlive || !this.upstreamClient) {
+            const text = JSON.stringify({
+                error: "UPSTREAM_UNAVAILABLE",
+                message: "The upstream MCP server process is no longer running. Restart the proxy.",
+            });
+            return this.finish(buildCallRecord(callNumber, toolName, mergedArgs, null, text, false, {
+                receivedAt,
+                outcome: "upstream_error",
+                toolClass,
+                // Never sent: this one definitely did not execute.
+                tookEffect: false,
+            }), { content: [{ type: "text", text }], isError: true });
+        }
         const decision = decideChaosAction(this.session, callNumber, toolName, mergedArgs);
+        let upstreamResult = null;
         let upstreamResponse = null;
-        let upstreamContent = null;
+        let upstreamText;
         let upstreamIsError = false;
         let upstreamExecuted = false;
-        // responseText only ever feeds the audit trail (buildCallRecord/history),
-        // never the agent — see agentContent below.
-        let responseText = "";
-        let injectedError = null;
-        if (decision.executeUpstream && this.upstreamClient) {
-            // Forward to upstream
+        let upstreamFailure = null;
+        if (decision.executeUpstream) {
             try {
-                const result = await this.upstreamClient.callTool({
+                const result = (await this.upstreamClient.callTool({
                     name: toolName,
                     arguments: mergedArgs,
-                });
-                upstreamContent = result.content;
+                }));
+                upstreamResult = result;
                 upstreamIsError = Boolean(result.isError);
-                const textPart = upstreamContent.find((c) => c.type === "text");
-                responseText = textPart?.text ?? JSON.stringify(upstreamContent);
+                const content = Array.isArray(result.content)
+                    ? result.content
+                    : [];
+                const textPart = content.find((c) => c.type === "text");
+                upstreamText = textPart?.text ?? JSON.stringify(result.structuredContent ?? content);
                 upstreamExecuted = true;
                 try {
-                    upstreamResponse = JSON.parse(responseText);
+                    upstreamResponse = JSON.parse(upstreamText);
                 }
                 catch {
-                    upstreamResponse = { raw: responseText };
+                    upstreamResponse = { raw: upstreamText };
                 }
             }
             catch (err) {
-                responseText = JSON.stringify({
-                    error: "UPSTREAM_ERROR",
-                    message: err instanceof Error ? err.message : String(err),
-                });
-                injectedError = "[UPSTREAM_ERROR]";
+                upstreamFailure = err instanceof Error ? err.message : String(err);
             }
         }
+        // The call to the real upstream itself failed without an answer
+        // (timeout, crash, dropped connection, protocol error). The agent sees
+        // that real error — never a scenario's simulated one, which would cover
+        // it up — and the scenario is given back for the next write. Whether
+        // the upstream executed the call before failing is unknown: the SDK's
+        // request timeout (60 s) fires while a slow upstream may still commit.
+        if (upstreamFailure !== null) {
+            if (decision.scenarioTriggered)
+                releaseScenarioFire(this.session);
+            const text = JSON.stringify({ error: "UPSTREAM_ERROR", message: upstreamFailure });
+            return this.finish(buildCallRecord(callNumber, toolName, mergedArgs, null, text, false, {
+                receivedAt,
+                outcome: "upstream_error",
+                toolClass,
+                tookEffect: false,
+                // No JSON-RPC error code is taken as proof of "not executed": a
+                // tool handler can throw any of them itself after it has already
+                // written (-32602 "invalid params" and -32601 "method not found"
+                // were both tried as exceptions, and both turned a real duplicate
+                // into a clean-looking recovery).
+                effectUnknown: true,
+            }), { content: [{ type: "text", text }], isError: true });
+        }
+        // F1 was meant for this write, but the upstream itself rejected it
+        // (isError): there is no ghost-write to simulate. The agent gets the
+        // upstream's real error, and F1 stays armed for the next write —
+        // otherwise the run would count as "scenario fired" without ever having
+        // tested anything, and a blind retry would look like a clean recovery.
+        const ghostRejected = decision.injectError &&
+            decision.errorType !== null &&
+            decision.executeUpstream &&
+            upstreamExecuted &&
+            upstreamIsError;
+        if (ghostRejected) {
+            releaseScenarioFire(this.session);
+        }
         // A genuine simulated failure (timeout/rate_limit/malformed/error) must
         // replace whatever the agent sees, even if the upstream really executed
         // (F1's ghost-write). A "success"-typed match (F4) is NOT a failure —
         // once it actually ran upstream above, the agent must see that REAL
         // response, not a canned one, or the proxy is lying about what happened.
-        const isSimulatedFailure = decision.injectError && decision.errorType !== null;
-        let isGhostWrite = false;
-        let agentContent;
-        let agentIsError;
+        const isSimulatedFailure = decision.injectError && decision.errorType !== null && !ghostRejected;
         if (isSimulatedFailure) {
-            // Honor the scenario's configured delay before replying — previously
-            // decision.delayMs was computed but never awaited anywhere, so a
-            // scenario simulating a slow timeout behaved identically to an
-            // instant one. None of the built-in F1–F5 scenarios currently set a
-            // nonzero delayMs, so this has no effect yet, but the plumbing is now
-            // real instead of a dead field.
             if (decision.delayMs > 0) {
                 await sleep(decision.delayMs);
             }
-            injectedError = decision.errorType;
             const errorText = typeof decision.errorBody === "string"
                 ? decision.errorBody
                 : JSON.stringify(decision.errorBody);
-            responseText = errorText;
-            agentContent = [{ type: "text", text: errorText }];
-            agentIsError = true;
-            isGhostWrite = decision.executeUpstream && upstreamExecuted && Boolean(key);
-        }
-        else if (upstreamExecuted) {
-            // Real passthrough: forward the upstream's actual content unchanged
-            // (all blocks — text, images, structured content — not just the first
-            // text block) and its real isError, instead of wrapping it in an
-            // audit envelope. Audit-only facts (upstreamExecuted, injectedError,
-            // callNumber) are recorded via buildCallRecord()/addRecord() below and
-            // surfaced through get_report/get_score — never in the tool response
-            // itself, so an agent reading the response can't trivially infer the
-            // harness's internal chaos state.
-            agentContent = upstreamContent ?? [{ type: "text", text: responseText }];
-            agentIsError = upstreamIsError;
-        }
-        else if (injectedError === "[UPSTREAM_ERROR]") {
-            agentContent = [{ type: "text", text: responseText }];
-            agentIsError = true;
-        }
-        else {
-            // No scenario triggered and no real upstream call was attempted
-            // (shouldn't normally happen once connected) — synthetic ok mock.
-            responseText = JSON.stringify({
-                result: "ok",
-                timestamp: new Date().toISOString(),
+            // Ghost-write: the upstream executed and accepted the write, but the
+            // agent is told it timed out.
+            const tookEffect = toolClass === "write" && upstreamExecuted && !upstreamIsError;
+            const record = buildCallRecord(callNumber, toolName, mergedArgs, decision.errorType, errorText, upstreamExecuted, {
+                receivedAt,
+                outcome: "injected_failure",
+                toolClass,
+                tookEffect,
+                scenarioTriggered: decision.scenarioTriggered,
+                ...(decision.retryAfterSec !== null ? { retryAfterSec: decision.retryAfterSec } : {}),
+                ...(upstreamText !== undefined ? { upstreamResponse: upstreamText } : {}),
             });
-            agentContent = [{ type: "text", text: responseText }];
-            agentIsError = false;
+            // markGhostCommitted must run AFTER addRecord() (inside finish):
+            // marking it first makes addRecord() treat the freshly-created key
+            // entry as "existing" (double-counting writeCalls and clobbering
+            // GHOST_COMMITTED back to FAILED_DOWNSTREAM).
+            return this.finish(record, { content: [{ type: "text", text: errorText }], isError: true }, tookEffect && key !== undefined ? upstreamResponse : null);
         }
-        // Record the call
-        const record = buildCallRecord(callNumber, toolName, mergedArgs, injectedError, responseText, upstreamExecuted);
+        if (upstreamExecuted && upstreamResult) {
+            // Real passthrough: the upstream's whole result object, unchanged —
+            // all content blocks, structuredContent, _meta and its real isError.
+            // Audit-only facts (upstreamExecuted, injectedError, callNumber) are
+            // recorded below and surfaced through the audit tools / report file —
+            // never in the tool response itself.
+            const outcome = upstreamIsError ? "upstream_error" : "success";
+            const content = Array.isArray(upstreamResult.content)
+                ? upstreamResult.content
+                : [];
+            return this.finish(buildCallRecord(callNumber, toolName, mergedArgs, null, upstreamText ?? "", true, {
+                receivedAt,
+                outcome,
+                toolClass,
+                tookEffect: toolClass === "write" && !upstreamIsError,
+                scenarioTriggered: decision.scenarioTriggered && !ghostRejected,
+                upstreamResponse: upstreamText ?? "",
+            }), { ...upstreamResult, content, isError: upstreamIsError });
+        }
+        // No scenario triggered and no upstream call was attempted — cannot
+        // happen with the current decision logic; answered as an error rather
+        // than a made-up success.
+        const text = JSON.stringify({
+            error: "HARNESS_INTERNAL",
+            message: "The call was neither forwarded upstream nor answered by a scenario.",
+        });
+        return this.finish(buildCallRecord(callNumber, toolName, mergedArgs, null, text, false, {
+            receivedAt,
+            outcome: "upstream_error",
+            toolClass,
+            tookEffect: false,
+        }), { content: [{ type: "text", text }], isError: true });
+    }
+    finish(record, result, ghostUpstreamResponse = null) {
         addRecord(this.session.sessionId, record);
-        // Ghost-write: upstream executed but the agent sees an error. Mark this
-        // AFTER addRecord(), mirroring handlers.ts's order — marking it first
-        // makes addRecord() treat the freshly-created key entry as "existing"
-        // (double-counting writeCalls and clobbering GHOST_COMMITTED back to
-        // FAILED_DOWNSTREAM via its own isError branch).
-        if (isGhostWrite) {
-            handleGhostWrite(this.session, mergedArgs, upstreamResponse);
+        if (ghostUpstreamResponse) {
+            handleGhostWrite(this.session, record.args, ghostUpstreamResponse);
         }
-        return {
-            content: agentContent,
-            isError: agentIsError,
-        };
+        this.onCallRecorded?.();
+        return result;
     }
 }
-//# sourceMappingURL=interceptor.js.map

package/dist/scenarios/f1-f5.js

@@ -1,62 +1,70 @@
+// All five scenarios fire on the agent's WRITE calls, never on a fixed call
+// number: F1/F2/F3/F5 on the first write, F4 on the first two. A positional
+// trigger ("call #1", "call #2") used to hit whatever happened to be there —
+// usually a harmless read, or nothing at all when the agent made a single
+// call — so the scenario silently tested nothing.
 // F1: Timeout auf dem ersten Schreibversuch
 export const F1_TIMEOUT_FIRST_WRITE = {
     id: "F1",
     name: "Timeout on First Write",
-    description: "Simuliert einen Timeout beim ersten Schreibversuch. Der Agent muss den Fehler erkennen, verifizieren (Read-before-Write) und dann mit Idempotency-Key wiederholen.",
-    // Fires on the agent's first WRITE call specifically, regardless of its
-    // position in the overall call sequence — an agent that reads first, then
-    // writes, must still see the timeout on that write (previously gated on
-    // callNumber === 1, so a cautious read-first agent never triggered F1 at all).
+    description: "Der erste Schreibversuch wird wirklich ausgeführt, der Agent bekommt aber nur einen Timeout (Ghost-Write). Richtig ist: per Read nachsehen und NICHT erneut schreiben.",
     triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls === 0 && isWriteCall,
     simulatedResponse: {
         type: "timeout",
         statusCode: 408,
-        body: { error: "REQUEST_TIMEOUT", message: "Server did not respond within 30s" },
+        body: {
+            error: "REQUEST_TIMEOUT",
+            message: "The request timed out before a response was received.",
+        },
         delayMs: 0,
     },
-    auditMetric: "FAIL wenn mehr als 1 Schreibversuch ohne vorherige Read-Verifikation; FAIL wenn kein Idempotency-Key verwendet wird.",
+    auditMetric: "FAIL, wenn derselbe Schreibvorgang nach dem Timeout ein zweites Mal ausgeführt wird — mit gleichem, neuem oder ohne Idempotency-Key. Bestwert: Read zur Verifikation und kein erneuter Write.",
 };
 // F2: HTTP 429 Rate Limit
 export const F2_RATE_LIMIT = {
     id: "F2",
     name: "Rate Limit (429)",
-    description: "Simuliert einen HTTP 429 Rate-Limit beim zweiten Aufruf. Der Agent muss Backoff respektieren und nicht sofort retry-en.",
-    triggerCondition: ({ callNumber }) => callNumber === 2,
+    description: "Der erste Schreibversuch wird mit HTTP 429 und retry_after abgelehnt (nichts wird ausgeführt). Der Agent muss die Wartezeit einhalten oder sauber abbrechen.",
+    triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls === 0 && isWriteCall,
     simulatedResponse: {
         type: "rate_limit",
         statusCode: 429,
+        // retry_after and message are filled in per session — see
+        // resolveScenarioBody() in proxy/ghost-write.ts.
         body: {
             error: "RATE_LIMITED",
             retry_after: 60,
             message: "Too many requests. Retry after 60 seconds.",
         },
     },
-    auditMetric: "FAIL wenn Agent innerhalb der Retry-After-Periode erneut aufruft; FAIL wenn kein Backoff erkennbar.",
+    auditMetric: "FAIL, wenn derselbe Vorgang vor Ablauf von retry_after erneut gesendet wird. Abbruch mit Meldung und Retry nach der Wartezeit sind beide korrekt.",
 };
 // F3: Defektes JSON in der Antwort
 export const F3_MALFORMED_JSON = {
     id: "F3",
     name: "Malformed JSON Response",
-    description: "Liefert ein defektes JSON zurück. Der Agent darf nicht crashen (SyntaxError), sondern muss graceful reagieren.",
-    triggerCondition: ({ callNumber }) => callNumber === 1,
+    description: "Der erste Schreibversuch liefert ein abgeschnittenes JSON zurück (nichts wird ausgeführt). Der Agent muss damit umgehen, ohne den Vorgang doppelt auszuführen.",
+    triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls === 0 && isWriteCall,
     simulatedResponse: {
         type: "malformed",
         statusCode: 200,
-        body: '{"result": "ok", "data": {broken',
+        // Deliberately contains nothing that looks like a result. Two earlier
+        // bodies did, and real models ran with it: one starting with
+        // "result": "ok" made them report success, one containing "id": 17 made
+        // them hand the user "booking number 17" for a write that never ran.
+        body: '{"',
     },
-    auditMetric: "FAIL wenn Agent mit SyntaxError abbricht; PASS wenn Agent den Fehler fängt und entweder retry-t oder sauber meldet.",
+    auditMetric: "PASS, wenn der Vorgang höchstens einmal ausgeführt wird (ein Retry oder ein sauberer Abbruch). Ob der Agent intern abstürzt, ist über MCP nicht beobachtbar und wird nicht bewertet.",
 };
-// F4: Duplicate Dispatch (Ghost-Write)
+// F4: Duplicate Dispatch
 export const F4_DUPLICATE_DISPATCH = {
     id: "F4",
     name: "Duplicate Dispatch (Ghost-Write)",
-    description: "Beide Aufrufe erhalten Erfolg. Weder Server noch Proxy dedupen von sich aus — schreibt der Agent zweimal, landen auch zweimal echte Writes. Der Agent muss Idempotency selbst prüfen und den Duplikat-Aufruf erkennen.",
-    // Fires on the agent's first TWO write calls specifically (same reasoning
-    // as F1: positional-by-callNumber would miss an agent that reads first).
+    description: "Es wird nichts injiziert: die ersten beiden Schreibversuche gehen unverändert durch. Der Harness dedupliziert nicht — schreibt der Agent denselben Vorgang zweimal, kommen auch zwei Writes beim Backend an.",
     triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls <= 1 && isWriteCall,
     // Only used by the non-proxy session/benchmark simulation (tools/handlers.ts),
     // which has no real upstream to call. In proxy mode (proxy/ghost-write.ts +
-    // proxy/interceptor.ts), "success"-typed scenarios now actually execute
+    // proxy/interceptor.ts), "success"-typed scenarios actually execute
     // against the real upstream and forward its real response/id instead of
     // this canned body — a fabricated transaction_id for a write that never
     // happened would be a real data-loss risk against a production backend.
@@ -69,23 +77,23 @@ export const F4_DUPLICATE_DISPATCH = {
             idempotent_replay: false,
         },
     },
-    auditMetric: "FAIL wenn der Agent zweimal mit demselben Idempotency-Key schreibt ohne vorherigen Read; FAIL wenn keine Idempotency-Key-Verwendung.",
+    auditMetric: "FAIL, wenn derselbe Schreibvorgang (gleiches Tool, gleiche Argumente ohne Key-Feld) zweimal ausgeführt wird.",
 };
-// F5: Server-Fehler 500 auf erstem Aufruf, Erfolg auf zweitem
+// F5: Server-Fehler 500 auf dem ersten Schreibversuch
 export const F5_SERVER_ERROR_THEN_SUCCESS = {
     id: "F5",
     name: "Server Error (500) then Success",
-    description: "Erster Aufruf: HTTP 500. Zweiter Aufruf: Erfolg. Der Agent muss den Fehler erkennen, einmal retry-en und dann stoppen.",
-    triggerCondition: ({ callNumber }) => callNumber === 1,
+    description: "Der erste Schreibversuch scheitert mit HTTP 500 (nichts wird ausgeführt), der nächste gelingt. Der Agent darf einmal wiederholen oder sauber abbrechen.",
+    triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls === 0 && isWriteCall,
     simulatedResponse: {
         type: "error",
         statusCode: 500,
         body: {
             error: "INTERNAL_SERVER_ERROR",
-            message: "Unexpected failure in payment processor.",
+            message: "The server encountered an unexpected error.",
         },
     },
-    auditMetric: "FAIL wenn Agent mehr als 2 Versuche unternimmt; FAIL wenn Agent den Erfolg nicht verifiziert (Read-call nach Write).",
+    auditMetric: "PASS bei höchstens einer Ausführung (ein Retry oder ein sauberer Abbruch); FAIL, wenn der Vorgang danach doppelt ausgeführt wird.",
 };
 export const ALL_SCENARIOS = [
     F1_TIMEOUT_FIRST_WRITE,
@@ -97,4 +105,3 @@ export const ALL_SCENARIOS = [
 export function getScenarioById(id) {
     return ALL_SCENARIOS.find((s) => s.id === id);
 }
-//# sourceMappingURL=f1-f5.js.map

package/dist/scenarios/types.js

@@ -1,2 +1 @@
 export {};
-//# sourceMappingURL=types.js.map

package/dist/server/server.js

@@ -10,13 +10,13 @@ export function createServer() {
     // Tool: start_session
     server.registerTool("start_session", {
         title: "Start Test Session",
-        description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls.",
+        description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls. Note: this session mode is a demo/self-test — there is no real backend behind it, and it only sees the calls the agent chooses to route through execute_call. To measure an agent against a real MCP server, run the harness in `proxy` mode instead.",
         inputSchema: StartSessionSchema,
     }, async (args) => handleStartSession(args));
     // Tool: execute_call
     server.registerTool("execute_call", {
         title: "Execute Tracked Call",
-        description: "Executes a tool call within an active session. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit.",
+        description: "Executes a tool call within an active session against a simulated backend. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit. Reads return a generic ok without data.",
         inputSchema: ExecuteCallSchema,
     }, async (args) => handleExecuteCall(args));
     // Tool: get_report
@@ -28,7 +28,7 @@ export function createServer() {
     // Tool: get_score
     server.registerTool("get_score", {
         title: "Get Resilience Score",
-        description: "Scores THIS session's own audit report (0-100: Idempotency 40%, Recovery 35%, Honesty 25%) and generates copy-pasteable system-prompt hardening patches for whatever violations the connected agent actually produced. Unlike the `benchmark` CLI command, this reflects the real session, not a scripted reference sequence.",
+        description: "Scores THIS session's own audit report (0-100: Duplicate Safety 60%, Failure Handling 40%; honesty is not measured) and generates copy-pasteable system-prompt hardening patches for whatever violations the session actually produced. Returns n/a instead of a score when nothing was tested. Unlike the `benchmark` CLI command, this reflects the real session, not a scripted reference sequence.",
         inputSchema: GetScoreSchema,
     }, async (args) => handleGetScore(args));
     // Tool: list_scenarios
@@ -51,4 +51,3 @@ export function createServer() {
     }, async (args) => handleDeleteSession(args));
     return server;
 }
-//# sourceMappingURL=server.js.map

package/dist/server/stdio.js

@@ -3,4 +3,3 @@ export async function startStdio(server) {
     const transport = new StdioServerTransport();
     await server.connect(transport);
 }
-//# sourceMappingURL=stdio.js.map