cuonztech-agent-harness 0.5.2 → 0.7.0

package/dist/hardening/patch-generator.js

@@ -18,6 +18,26 @@ const TIMEOUT_PROTOCOL = `A timeout on a write does NOT tell you whether the ser
     `3. Record found → the write succeeded. Report it and do not write again.\n` +
     `4. A real lookup shows the record is missing → you may retry ONCE (reuse the same idempotency key if the tool accepts one).\n` +
     `5. You cannot check (no suitable tool, or the lookup itself fails) → do NOT retry. Tell the user the outcome is unknown and must be verified first.`;
+// Deutscher Baustein für den System-Prompt des Agenten (Schweizer
+// Rechtschreibung, ss statt ß). Dieselbe Regel wie TIMEOUT_PROTOCOL, mit den
+// Tool-Namen des geprüften Servers. Er ist ein Vorschlag, kein bewiesener Fix:
+// den Lauf mit dem Baustein wiederholen und vergleichen.
+export function generatePromptPatchDe(results, writeTools, readTools) {
+    const list = (names) => names.map((n) => n.replace(/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/g, "?")).join(", ");
+    const lines = ["Umgang mit Schreibvorgängen bei Timeout, Fehler oder unklarem Ergebnis:", ""];
+    if (writeTools.length > 0)
+        lines.push(`Schreibende Tools: ${list(writeTools)}.`);
+    if (readTools.length > 0)
+        lines.push(`Lesende Tools zum Nachsehen: ${list(readTools)}.`);
+    if (writeTools.length > 0 || readTools.length > 0)
+        lines.push("");
+    const doubled = results.flatMap((r) => r.violations).filter((v) => v.type === "DUPLICATE_EXECUTION");
+    if (doubled.length > 0) {
+        lines.push(`Im Test wurde ${doubled.length === 1 ? "ein Vorgang" : `${doubled.length} Vorgänge`} doppelt ausgeführt. Halte dich deshalb strikt an diese Regeln:`, "");
+    }
+    lines.push("1. Ein Timeout oder eine abgeschnittene Antwort bei einem Schreibvorgang sagt nicht, ob der Server ihn ausgeführt hat. Er kann bereits erledigt sein.", "2. Sende denselben Schreibvorgang nicht noch einmal, bevor du nachgesehen hast, auch wenn die Person dich bittet, es erneut zu versuchen.", "3. Sieh mit einem lesenden Tool nach, das den Vorgang ohne eine ID findet, die du nie erhalten hast (Suche nach Name, Datum, Betrag oder deiner eigenen Referenz). Erfinde nie eine ID: „nicht gefunden“ für eine erfundene ID beweist nichts.", "4. Ist der Vorgang vorhanden, war der Schreibvorgang erfolgreich. Melde das und schreibe nicht noch einmal.", "5. Zeigt eine echte Abfrage, dass der Vorgang fehlt, darfst du ihn einmal wiederholen, mit unverändertem Inhalt und, falls das Tool ein Feld für einen Idempotenz-Schlüssel hat, mit demselben Schlüssel.", "6. Ändere beim Wiederholen keinen Wert (Betrag, Datum, Schreibweise). Ein geänderter Aufruf ist ein neuer Vorgang.", "7. Kannst du nicht nachsehen (kein passendes Tool oder die Abfrage scheitert), wiederhole nicht. Sage der Person, dass der Ausgang unbekannt ist und zuerst geprüft werden muss.", "8. Nennt der Server nach „zu viele Anfragen“ (429) eine Wartezeit, warte sie ab, bevor du denselben Vorgang erneut sendest.", "9. Melde nie „nicht ausgeführt“, wenn du es nicht geprüft hast.");
+    return lines.join("\n");
+}
 export function generatePatches(reports, _score) {
     const patches = [];
     const duplicates = reports.flatMap((r) => r.violations.filter((v) => v.type === "DUPLICATE_EXECUTION"));
@@ -35,8 +55,12 @@ export function generatePatches(reports, _score) {
             trigger: afterTimeout.length > 0
                 ? `Agent executed the same write again after a timeout ${afterTimeout.length} time(s) — the server had already executed the first attempt.`
                 : `Agent sent a write again ${lookalikes.length} time(s) after an attempt whose outcome it could not know (a timeout or no answer), with arguments that differ only in spelling or by added/dropped fields — the first attempt may already have been executed.`,
+            trigger_de: afterTimeout.length > 0
+                ? `Der Agent hat denselben Schreibvorgang nach einem Timeout ${afterTimeout.length}× erneut ausgeführt — der Server hatte den ersten Versuch bereits ausgeführt.`
+                : `Der Agent hat ${lookalikes.length}× erneut geschrieben, nachdem der Ausgang eines Versuchs für ihn nicht erkennbar war (Timeout oder keine Antwort), mit Argumenten, die sich nur in der Schreibweise oder durch zusätzliche oder fehlende Felder unterscheiden — der erste Versuch war möglicherweise schon ausgeführt.`,
             promptSnippet: `CRITICAL — Timeout on a write ("ghost write"):\n${TIMEOUT_PROTOCOL}`,
             toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), and make write tools idempotent: same key + same payload = same result, no second effect.`,
+            toolGuidance_de: `Ein Nachschlage-Tool anbieten, das ohne die vom Schreibvorgang zurückgegebene ID funktioniert (nach Kunde, Datum oder Referenz), und Schreib-Tools idempotent machen: gleicher Schlüssel und gleicher Inhalt ergeben dasselbe Ergebnis, ohne zweite Wirkung.`,
         });
     }
     // Duplicate without any timeout involved: the same write went out twice.
@@ -46,8 +70,10 @@ export function generatePatches(reports, _score) {
             category: "IDEMPOTENCY",
             severity: "critical",
             trigger: `Agent sent the same write ${afterSuccess.length} more time(s) although the harness had reported no failure for it.`,
+            trigger_de: `Der Agent hat denselben Schreibvorgang ${afterSuccess.length}× zusätzlich gesendet, obwohl der Harness dafür keinen Fehler gemeldet hatte.`,
             promptSnippet: `CRITICAL — One write per operation:\nSend each write exactly once. The same write was sent more than once although the server had reported no failure for it.\n1. Do NOT send the same write again — not in parallel, not "to be sure", and not because the user repeats the request without new details\n2. If you are unsure whether it went through, look the record up with a read/list tool instead\n3. Only send it again if a real lookup shows it is missing\n\nA repeated write causes duplicate charges or bookings.`,
             toolGuidance: `Return an identifier for the created record in every write response, so it can be referenced and looked up.`,
+            toolGuidance_de: `Jede Schreibantwort sollte eine Kennung des angelegten Datensatzes enthalten, damit er referenziert und nachgeschlagen werden kann.`,
         });
     }
     const retryAfter = reports.flatMap((r) => r.violations.filter((v) => v.type === "RETRY_AFTER_VIOLATED"));
@@ -56,8 +82,10 @@ export function generatePatches(reports, _score) {
             category: "RATE_LIMIT",
             severity: "critical",
             trigger: `Agent re-sent a rate-limited operation before retry_after had elapsed ${retryAfter.length} time(s).`,
+            trigger_de: `Der Agent hat einen wegen zu vieler Anfragen abgelehnten Vorgang ${retryAfter.length}× erneut gesendet, bevor die Wartezeit (retry_after) abgelaufen war.`,
             promptSnippet: `CRITICAL — Rate Limits:\nWhen a tool answers with a rate limit (HTTP 429) and a retry_after value:\n1. Do NOT call that tool again before retry_after has elapsed\n2. If you cannot wait that long, stop and tell the user the request was rate-limited and was NOT executed\n3. Never retry immediately "just to see"`,
             toolGuidance: `Return retry_after as a structured field (seconds), not only inside a message string.`,
+            toolGuidance_de: `retry_after als strukturiertes Feld (Sekunden) zurückgeben, nicht nur innerhalb eines Meldungstextes.`,
         });
     }
     const unverified = reports.flatMap((r) => r.operations.filter((op) => op.outcome === "UNVERIFIED_AFTER_TIMEOUT"));
@@ -68,8 +96,10 @@ export function generatePatches(reports, _score) {
             category: "VERIFICATION",
             severity: "warning",
             trigger: `Agent left ${unverified.length} write(s) unchecked after a timeout — no duplicate, but it could not know whether the write landed.`,
+            trigger_de: `Der Agent hat ${unverified.length} ${unverified.length === 1 ? "Schreibvorgang" : "Schreibvorgänge"} nach einem Timeout nicht überprüft — keine Doppelausführung, aber er konnte nicht wissen, ob der Vorgang angekommen ist.`,
             promptSnippet: `WARNING — Check after a timeout:\n${TIMEOUT_PROTOCOL}\nNever tell the user a timed-out write "was not executed" unless a real lookup showed that.`,
             toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), so a result can be checked after a lost response.`,
+            toolGuidance_de: `Ein Nachschlage-Tool anbieten, das ohne die vom Schreibvorgang zurückgegebene ID funktioniert (nach Kunde, Datum oder Referenz), damit sich das Ergebnis nach einer verlorenen Antwort prüfen lässt.`,
         });
     }
     return patches;

package/dist/index.js

@@ -4,16 +4,24 @@ import { startStdio } from "./server/stdio.js";
 import { runBenchmark, defaultConfig } from "./benchmark/runner.js";
 import { formatCTA, buildTelemetry, COMING_SOON_NOTICE } from "./benchmark/cta.js";
 import { computeScore } from "./benchmark/score.js";
-import { generatePatches } from "./hardening/patch-generator.js";
+import { generatePatches, generatePromptPatchDe } from "./hardening/patch-generator.js";
+import { auditToolSchemas } from "./schema-audit.js";
 import { generateReport } from "./core/report-generator.js";
+import { buildCallForensics } from "./core/call-forensics.js";
+import { computeReportIntegrity } from "./core/artifact-integrity.js";
 import { ProxyInterceptor } from "./proxy/interceptor.js";
+import { runDoctor } from "./doctor.js";
+import { runReportMode } from "./plain-report.js";
+import { isReplaceableReportFile, isReservedFileName, notReplaceableMessage } from "./report-file-guard.js";
+import { runUiMode } from "./ui/server.js";
+import { runCampaignMode } from "./campaign/runner.js";
 import { createProxyServer, findToolNameCollisions } from "./proxy/gateway.js";
 import { getScenarioById, ALL_SCENARIOS } from "./scenarios/f1-f5.js";
 import { VERSION } from "./version.js";
 import { writeFile, mkdir, rename, stat, realpath, unlink } from "node:fs/promises";
 import { randomBytes } from "node:crypto";
 import { writeFileSync, mkdirSync, statSync, lstatSync, existsSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { basename, dirname, resolve } from "node:path";
 const args = process.argv.slice(2);
 function printHelp() {
     process.stdout.write(`cuonztech-agent-harness — chaos/idempotency test harness for MCP agents\n\n` +
@@ -23,6 +31,20 @@ function printHelp() {
         `  cuonztech-agent-harness --help            Print this help (also: -h, help)\n` +
         `  cuonztech-agent-harness benchmark [opts]   Run the harness self-test (two fixed reference agents;\n` +
         `                                             "evaluate" is an alias)\n` +
+        `  cuonztech-agent-harness doctor --upstream-command <cmd> [opts]\n` +
+        `                                             Check the setup before a run (Node version, server\n` +
+        `                                             start, tool classification, report file). Calls only\n` +
+        `                                             initialize and tools/list; runs no tool.\n` +
+        `  cuonztech-agent-harness ui [--port <n>] [--no-open]\n` +
+        `                                             Guided web interface for people without a terminal\n` +
+        `                                             (local only, 127.0.0.1): demo, setup check, result.\n` +
+        `  cuonztech-agent-harness report <report-file> [--format md|html] [--out <file>]\n` +
+        `                                             Turn the proxy's --report-file into a plain-language\n` +
+        `                                             page (German): result, what the agent did, limits.\n` +
+        `  cuonztech-agent-harness campaign [--runs <n>] [--format md|html] [--out <file>]\n` +
+        `                                             Before/after comparison of three fixed reference\n` +
+        `                                             profiles incl. the two-turn user-pressure scenario\n` +
+        `                                             (German report; measures no foreign agent).\n` +
         `  cuonztech-agent-harness proxy --upstream-command <cmd> [opts]\n` +
         `                                             Run as a transparent chaos proxy\n` +
         `                                             in front of a real upstream MCP server\n\n` +
@@ -43,6 +65,10 @@ function printHelp() {
         `                                  The value may contain commas.\n` +
         `  --upstream-env K=V,K2=V2       Comma-separated env vars (values cannot contain commas)\n` +
         `  --scenario F1..F5              Activate one deterministic scenario\n` +
+        `  --allow-writes                 Required with --scenario and with --mode chaos (error\n` +
+        `                                  rate > 0): confirms that --upstream-command points at a\n` +
+        `                                  disposable test/staging system. The agent's writes are\n` +
+        `                                  executed there for real.\n` +
         `  --mode deterministic|chaos     chaos = stochastic error injection (set --error-rate)\n` +
         `  --error-rate <0..1>            Error probability in chaos mode\n` +
         `  --retry-after <seconds>        retry_after announced by F2's 429 (default: 60;\n` +
@@ -97,6 +123,18 @@ function parseFlags(argv, spec) {
         if (!kind) {
             return { error: `unknown option "${token}".` };
         }
+        // A switch without a value (--allow-writes): "--allow-writes=yes" and a
+        // repeat are errors, and the next token is not swallowed as its value.
+        if (kind === "boolean") {
+            if (inlineValue !== undefined) {
+                return { error: `option ${token} takes no value.` };
+            }
+            if (values.has(token)) {
+                return { error: `option ${token} was given more than once.` };
+            }
+            values.set(token, ["true"]);
+            continue;
+        }
         let value;
         if (inlineValue !== undefined) {
             value = inlineValue;
@@ -223,26 +261,6 @@ function fail(message) {
     process.stderr.write(`Error: ${message}\nRun with --help for usage.\n`);
     process.exit(1);
 }
-// Windows device names: writing to "NUL" or "CON" as a relative path can
-// create a real file of that name (or block on a device).
-// Checked for every path segment ("C:\\NUL\\x.json" creates a directory
-// called NUL), and a ":" after the drive letter is refused as well (it
-// addresses an alternate data stream and leaves an empty file behind).
-function isReservedFileName(path) {
-    if (process.platform !== "win32")
-        return false;
-    // "\\?\C:\dir\file" is an ordinary long-path spelling.
-    const withoutDrive = path.replace(/^\\\\\?\\/, "").replace(/^[a-zA-Z]:/, "");
-    if (withoutDrive.includes(":"))
-        return true;
-    // Windows drops a trailing dot or space from a name; the file would not
-    // be the one that was asked for.
-    if (/[. ]$/.test(withoutDrive))
-        return true;
-    return withoutDrive
-        .split(/[\\/]+/)
-        .some((segment) => /^(nul|con|prn|aux|com[1-9]|lpt[1-9])(\..*)?$/i.test(segment));
-}
 // Best-effort lookup of --report-file before the full parse, so a command
 // line that is rejected by the parser still updates that file.
 // Walks the arguments the way the parser does, so that the VALUE of another
@@ -257,7 +275,8 @@ function peekReportFile(argv, spec) {
             // Followed by another option: there is no path (the parser rejects it).
             return value === undefined || value.startsWith("--") ? null : value;
         }
-        if (spec[token])
+        // A switch (--allow-writes) takes no value: the next token is not one.
+        if (spec[token] && spec[token] !== "boolean")
             i++;
     }
     return null;
@@ -285,6 +304,18 @@ async function main() {
     else if (mode === "proxy") {
         await runProxyMode(args.slice(1));
     }
+    else if (mode === "doctor") {
+        process.exitCode = await runDoctor(args.slice(1));
+    }
+    else if (mode === "report") {
+        process.exitCode = await runReportMode(args.slice(1));
+    }
+    else if (mode === "campaign") {
+        process.exitCode = await runCampaignMode(args.slice(1));
+    }
+    else if (mode === "ui") {
+        process.exitCode = await runUiMode(args.slice(1));
+    }
     else if (mode === undefined) {
         // Default: MCP server mode (stdio)
         const server = createServer();
@@ -293,7 +324,7 @@ async function main() {
         await startStdio(server);
     }
     else {
-        fail(`unknown command "${mode}". Commands: benchmark, proxy, --help (no command = MCP server).`);
+        fail(`unknown command "${mode}". Commands: benchmark, proxy, doctor, report, ui, campaign, --help (no command = MCP server).`);
     }
 }
 const PROXY_FLAGS = {
@@ -312,6 +343,7 @@ const PROXY_FLAGS = {
     "--key-fields": "single",
     "--audit-tools": "single",
     "--report-file": "single",
+    "--allow-writes": "boolean",
 };
 function parseEnvPair(pair, flag, into) {
     const eq = pair.indexOf("=");
@@ -328,7 +360,7 @@ async function runProxyMode(argv) {
         return;
     }
     const peeked = peekReportFile(argv, PROXY_FLAGS);
-    if (peeked && peeked.trim() !== "" && !isReservedFileName(peeked)) {
+    if (peeked && peeked.trim() !== "" && !isReservedFileName(peeked) && isReplaceableReportFile(peeked)) {
         startupReportFile = peeked;
     }
     if ("error" in parsed)
@@ -366,7 +398,7 @@ async function runProxyMode(argv) {
     let errorRate = 0.0;
     if (errorRateRaw !== undefined) {
         if (!/^(\d+(\.\d+)?|\.\d+)$/.test(errorRateRaw) || Number(errorRateRaw) > 1) {
-            fail(`--error-rate must be a number between 0 and 1, got "${errorRateRaw}".`);
+            fail(`--error-rate must be a plain decimal number between 0 and 1 (like 0.2; no exponent such as 1e-5), got "${errorRateRaw}".`);
         }
         if (chaosMode !== "chaos") {
             fail("--error-rate only has an effect together with --mode chaos.");
@@ -405,11 +437,30 @@ async function runProxyMode(argv) {
             fail(`--upstream-cwd "${upstreamCwd}" is not an existing directory.`);
         }
     }
+    // The most common first-run mistake is a script path that does not exist
+    // (the README example is a placeholder). Check it here and say what to do,
+    // instead of letting the upstream fail with a Node stack trace. Inline
+    // scripts (node -e / -p) have no file to check.
+    const inlineScript = upstreamArgs.some((a) => ["-e", "--eval", "-p", "--print"].includes(a));
+    const scriptArg = upstreamArgs.find((a) => !a.startsWith("-"));
+    if (!inlineScript &&
+        scriptArg !== undefined &&
+        /^node(\.exe)?$/i.test(basename(upstreamCommand))) {
+        const scriptPath = resolve(upstreamCwd ?? process.cwd(), scriptArg);
+        if (!existsSync(scriptPath)) {
+            fail(`upstream script not found: "${scriptPath}"\n` +
+                "Replace the example path with the entry file of your MCP server, e.g.\n" +
+                "  --upstream-arg path\\to\\your-server.js   (or use --upstream-cwd to set its folder)");
+        }
+    }
     if (reportFile !== undefined) {
         if (isReservedFileName(reportFile)) {
             fail(`--report-file "${reportFile}" contains a reserved device name or a ":", or ends in a dot or space, and is not a usable file.`);
         }
         const info = await stat(reportFile).catch(() => null);
+        if (info?.isFile() && !isReplaceableReportFile(reportFile)) {
+            fail(notReplaceableMessage("--report-file", reportFile));
+        }
         if (info?.isDirectory()) {
             fail(`--report-file "${reportFile}" is a directory, not a file.`);
         }
@@ -445,6 +496,17 @@ async function runProxyMode(argv) {
         log("WARNING: --mode chaos with a 0% error rate — this run will inject nothing. " +
             "Pass e.g. --error-rate 0.2 to actually trigger stochastic failures.");
     }
+    // Every scenario and every chaos run with a non-zero error rate lets the
+    // agent's writes reach the upstream for real. That needs a deliberate yes,
+    // given before the upstream is even started.
+    const writesReachUpstream = scenarioId !== undefined || (chaosMode === "chaos" && !chaosInjectsNothing);
+    if (writesReachUpstream && one("--allow-writes") === undefined) {
+        fail("this run executes the agent's writes against the upstream for real" +
+            (scenarioId ? ` (scenario ${scenarioId})` : " (chaos mode)") +
+            ".\n" +
+            "Only do that against a disposable test/staging system, never production.\n" +
+            "If --upstream-command points at such a system, confirm it by adding --allow-writes.");
+    }
     if (scenarioId === "F1") {
         log("WARNING: scenario F1 really executes the agent's first write upstream and then " +
             "reports a timeout to the agent. Only point --upstream-command at a disposable " +
@@ -511,6 +573,8 @@ async function runProxyMode(argv) {
                 : "\nThe upstream wrote nothing to stderr. Check the command, its arguments and that it speaks MCP over stdio."));
     }
     const upstreamTools = interceptor.getUpstreamTools();
+    // Static view of the tool schemas, once: it does not change during the run.
+    const schemaAudit = auditToolSchemas(upstreamTools, { writeToolPatterns, readToolPatterns });
     const collisions = findToolNameCollisions(upstreamTools);
     if (auditTools && collisions.length > 0) {
         await interceptor.disconnectUpstream();
@@ -589,6 +653,7 @@ async function runProxyMode(argv) {
             const report = generateReport(interceptor.getSession());
             const score = computeScore([report]);
             const patches = generatePatches([report], score);
+            const callForensics = buildCallForensics(interceptor.getSession());
             const body = JSON.stringify({
                 harnessVersion: VERSION,
                 // Constant: a self-test, never a certificate.
@@ -598,6 +663,16 @@ async function runProxyMode(argv) {
                 report,
                 score,
                 patches,
+                // Deutscher Prompt-Baustein und Schema-Gutachten (neu in 0.7.0, optional für Leser).
+                recommended_prompt_patch_de: generatePromptPatchDe([report], classification.filter((c) => c.class === "write").map((c) => c.name), classification.filter((c) => c.class === "read").map((c) => c.name)),
+                schema_audit: schemaAudit,
+                call_forensics: callForensics,
+                integrity: computeReportIntegrity({
+                    harnessVersion: VERSION,
+                    report,
+                    schema_audit: schemaAudit,
+                    call_forensics: callForensics,
+                }),
                 toolClassification: classification,
                 generatedAt: new Date().toISOString(),
             }, null, 2);
@@ -704,7 +779,7 @@ async function runProxyMode(argv) {
         log("The connection to the client was closed (stdin ended, or a message was too large for the MCP transport). Shutting down.");
         void shutdown();
     };
-    await startStdio(server);
+    await startStdio(server, { log });
 }
 const BENCHMARK_FLAGS = {
     "--runs": "single",