cuonztech-agent-harness 0.8.0 → 0.9.0

package/dist/plain-report.js

@@ -82,6 +82,10 @@ export const OUTCOMES = {
         text: "Es bleibt unklar, ob der Vorgang ausgeführt wurde. Weder als sicher bestätigt noch als doppelt nachgewiesen; nicht bewertet.",
     },
     DUPLICATE_EXECUTION: { good: false, text: "Der Vorgang wurde mehrfach ausgeführt." },
+    DEDUPLICATED_RETRY: {
+        good: true,
+        text: "Der Vorgang wurde nochmals gesendet, das Zielsystem hat die Wiederholung aber selbst erkannt: Die Wirkungs-Sonde hat keinen zweiten Datensatz gezählt.",
+    },
     RETRY_AFTER_VIOLATED: { good: false, text: "Der Agent hat zu früh erneut gesendet, vor Ablauf der angegebenen Wartezeit." },
 };
 export const LIMITS = [
@@ -171,6 +175,30 @@ function claimSection(claim) {
         lines.push("");
     return lines;
 }
+const PROBE_CONCLUSION_DE = {
+    confirmed: "bestätigt: Jeder Schreibversuch hat einen Datensatz erzeugt.",
+    deduplicated: "dedupliziert: Wiederholte oder gescheiterte Schreibvorgänge haben keinen neuen Datensatz erzeugt. Die Bewertung rechnet mit Datensätzen statt mit Ausführungen.",
+    partial: "teilweise: Einige Wiederholungen haben einen Datensatz erzeugt, andere nicht. Die Zählung kann nicht sagen, welcher Vorgang welchen erzeugt hat; die Bewertung rechnet mit den gesehenen Ausführungen.",
+    inconclusive: "nicht verwertbar: Die Bewertung rechnet mit den gesehenen Ausführungen.",
+    no_writes: "kein Schreibvorgang erreichte das Zielsystem, es gab nichts zu vergleichen.",
+};
+// Wirkungs-Sonde (--effect-probe): was gezählt wurde und was daraus folgt. Ohne
+// Sonde leer.
+export function effectProbeLinesDe(report) {
+    const p = report?.effectProbe;
+    if (!p)
+        return [];
+    const count = p.before === null || p.after === null ? "Zählung nicht verfügbar" : `Datensätze vorher ${p.before}, nachher ${p.after}, neu ${p.delta}`;
+    const lines = [
+        `Der Harness hat das Lese-Tool \`${p.tool.replace(/[`\r\n]/g, "?")}\` selbst aufgerufen und Datensätze gezählt: ${count}. ${p.attempts} Schreibversuch(e) in ${p.operations} Vorgang/Vorgängen erreichten das Zielsystem.`,
+        `Schluss: ${PROBE_CONCLUSION_DE[p.conclusion]}`,
+    ];
+    if (p.conclusion === "inconclusive" || p.conclusion === "partial") {
+        lines.push(`Begründung (Originaltext): ${p.note.replace(/[\r\n]+/g, " ")}`);
+    }
+    lines.push("Annahmen der Sonde: Jeder wirkende Schreibvorgang der gezählten Tools erzeugt genau einen Datensatz in dem, was die Sonde zählt; niemand sonst ändert diese Zahl während des Laufs; eine Erfolgsantwort bedeutet, dass ein Datensatz existiert. Sie sagt nichts über die Absicht des Agenten.");
+    return lines;
+}
 export function renderMarkdown(file, options = {}) {
     const pro = options.pro ?? true;
     const { headline, explanation } = verdictText(file);
@@ -209,6 +237,10 @@ export function renderMarkdown(file, options = {}) {
             }
             lines.push("");
         }
+        const probeLines = effectProbeLinesDe(report);
+        if (probeLines.length > 0) {
+            lines.push("## Wirkungs-Sonde", "", ...probeLines.map((l) => `- ${l}`), "");
+        }
         if (file.score) {
             lines.push("## Kennzahl", "");
             if (file.score.overall === null || file.score.overall === undefined) {

package/dist/proxy/classification-doubt.js

@@ -0,0 +1,89 @@
+const MAX_DESCRIPTION = 600;
+// A verb counts only where a verb stands in a description: at the start of a
+// clause ("Books a room"), or after a comma or a conjunction ("... and bills the
+// first night", ", or creates one"). The same words elsewhere are mostly nouns
+// ("Lists issues", "Returns charges", "the booking that updates the view").
+// Third person singular present (German: present); the bare form only at the
+// start ("Create a booking").
+const WRITE_VERBS = new Set([
+    "creates", "books", "bills", "charges", "sends", "deletes", "removes", "cancels", "updates",
+    "modifies", "writes", "inserts", "submits", "pays", "transfers", "refunds", "stores", "saves",
+    "persists", "reserves", "registers", "approves", "issues", "withdraws", "deposits", "purchases",
+    "places", "confirms", "schedules", "posts", "adds", "executes", "triggers", "publishes", "logs",
+    "sets", "notifies", "emails", "uploads", "generates", "archives", "overwrites", "increments", "marks",
+    "erstellt", "bucht", "sendet", "löscht", "loescht", "storniert", "ändert", "aendert", "schreibt",
+    "speichert", "zahlt", "belastet", "verrechnet", "überweist", "ueberweist", "registriert", "entfernt",
+    "schickt", "versendet", "aktualisiert", "reserviert", "bestätigt", "bestaetigt", "setzt", "erzeugt",
+    "überschreibt", "ueberschreibt", "verbucht",
+]);
+const BARE_WRITE_VERBS = new Set([
+    "create", "book", "bill", "charge", "send", "delete", "remove", "cancel", "update", "modify",
+    "write", "insert", "submit", "pay", "transfer", "refund", "store", "save", "persist", "reserve",
+    "register", "approve", "issue", "withdraw", "deposit", "purchase", "place", "confirm", "schedule",
+    "post", "add", "execute", "trigger", "publish", "log", "set", "notify", "email", "upload", "generate",
+    "archive", "overwrite", "increment", "mark",
+]);
+const CONJUNCTIONS = new Set(["and", "or", "then", "also", "und", "oder", "dann", "sowie", "danach", "plus"]);
+// A verb inside a negated clause says the opposite ("Never writes or sends anything",
+// "Sendet nichts, löscht nichts").
+const NEGATIONS = new Set([
+    "not", "never", "no", "without", "doesn't", "don't", "won't", "nothing", "nobody",
+    "nicht", "kein", "keine", "keinen", "ohne", "nie", "niemals", "nichts",
+]);
+const NEGATION_AFTER = 3; // "bucht nichts", "does nothing"
+function tokenize(clause) {
+    // Words and commas; letters by Unicode class (\b knows only ASCII).
+    return clause.toLowerCase().match(/[\p{L}\p{N}']+|,/gu) ?? [];
+}
+function describesWriting(description) {
+    const text = description.slice(0, MAX_DESCRIPTION).replace(/\s+/g, " ");
+    for (const clause of text.split(/[.;!?]+/)) {
+        const tokens = tokenize(clause);
+        for (let i = 0; i < tokens.length; i++) {
+            const word = tokens[i];
+            const prev = i > 0 ? tokens[i - 1] : null;
+            const inVerbPosition = prev === null || prev === "," || CONJUNCTIONS.has(prev);
+            const isVerb = inVerbPosition && (WRITE_VERBS.has(word) || (i === 0 && BARE_WRITE_VERBS.has(word)));
+            if (!isVerb)
+                continue;
+            const negated = tokens.slice(0, i).some((t) => NEGATIONS.has(t)) ||
+                tokens.slice(i + 1, i + 1 + NEGATION_AFTER).some((t) => NEGATIONS.has(t));
+            if (!negated)
+                return word;
+        }
+    }
+    return null;
+}
+// Unambiguous statements only, and only in the FIRST sentence ("The id field is
+// read-only" in a later sentence is about a parameter, not about the tool).
+const READ_ONLY_PHRASES = [
+    "read-only", "read only", "readonly", "only reads", "no side effect", "no side-effect",
+    "does not modify any", "does not change any", "does not write", "nur lesend", "schreibgeschützt",
+    "schreibgeschuetzt", "nur lesen",
+];
+function firstSentence(description) {
+    const text = description.slice(0, MAX_DESCRIPTION).replace(/\s+/g, " ").trim();
+    const end = text.search(/[.!?](\s|$)/);
+    return (end === -1 ? text : text.slice(0, end + 1)).toLowerCase();
+}
+export function classificationDoubt(tool, cls, source) {
+    const description = tool.description;
+    if (!description)
+        return undefined;
+    const nameBased = source === "default write pattern" || source === "read prefix";
+    if (!nameBased)
+        return undefined;
+    if (cls === "read") {
+        const verb = describesWriting(description);
+        return verb
+            ? `taken for a read by its name, but its description says it "${verb}" — if it writes, duplicates through it stay invisible (set --write-tools or the readOnlyHint annotation)`
+            : undefined;
+    }
+    if (cls === "write") {
+        const first = firstSentence(description);
+        if (READ_ONLY_PHRASES.some((p) => first.includes(p))) {
+            return "taken for a write by its name, but its description says it only reads — calls to it may be treated as writes (set --read-tools or the readOnlyHint annotation)";
+        }
+    }
+    return undefined;
+}

package/dist/proxy/gateway.js

@@ -86,6 +86,7 @@ export function createProxyServer(interceptor, options = {}) {
                     isError: true,
                 };
             }
+            await interceptor.refreshEffectProbe(true);
             const withSession = { session_id: interceptor.sessionId, format };
             if (name === `${AUDIT_TOOL_PREFIX}get_report`) {
                 return handleGetReport(withSession, { persist: false });

package/dist/proxy/interceptor.js

@@ -6,6 +6,8 @@
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { VERSION } from "../version.js";
+import { classificationDoubt } from "./classification-doubt.js";
+import { parseProbeReading } from "../core/effect-probe.js";
 import { readFileSync, readdirSync } from "node:fs";
 import { createSession, incrementCall, addRecord, classifyTool, extractKey, isKeyField, isReadTool, matchesAnyGlob, releaseScenarioFire, DEFAULT_WRITE_TOOL_PATTERNS, } from "../engine/state-machine.js";
 import { decideChaosAction, handleGhostWrite, buildCallRecord, } from "./ghost-write.js";
@@ -17,6 +19,11 @@ function sleep(ms) {
 // Anything left over stays "unclassified" — and is reported as such instead
 // of being quietly treated as a harmless read.
 export function classifyUpstreamTool(tool, config) {
+    const result = classifyByRules(tool, config);
+    const doubt = classificationDoubt(tool, result.class, result.source);
+    return doubt ? { ...result, doubt } : result;
+}
+function classifyByRules(tool, config) {
     const name = tool.name;
     const explicitWrite = config.writeToolPatterns ?? [];
     const explicitRead = config.readToolPatterns ?? [];
@@ -51,6 +58,7 @@ export function classifyUpstreamTool(tool, config) {
     return { name, class: "unclassified", source: "no annotation, no pattern match" };
 }
 const STDERR_TAIL_CHARS = 2000;
+const PROBE_TIMEOUT_MS = 15_000;
 const MAX_TOOL_LIST_PAGES = 1000;
 // Linux only (/proc). [ppid, starttime] of a process, or null if it is gone.
 function procStat(pid) {
@@ -123,6 +131,11 @@ export class ProxyInterceptor {
     onCallRecorded = null;
     // Called once if the upstream connection drops while the proxy is running.
     onUpstreamLost = null;
+    // --effect-probe: called when a background reading of the probe has landed,
+    // so the report file can be written again with it.
+    onProbeRefreshed = null;
+    probeTimer = null;
+    probeInFlight = null;
     constructor(config) {
         this.config = config;
         this.session = createSession(config.scenarioId ?? null, config.mode ?? "deterministic", config.errorRate ?? 0.0, config.writeToolPatterns, {
@@ -204,6 +217,9 @@ export class ProxyInterceptor {
         this.toolClassification = this.upstreamTools.map((tool) => classifyUpstreamTool(tool, this.config));
         for (const entry of this.toolClassification) {
             this.session.toolClassOverrides.set(entry.name, entry.class);
+            if (entry.doubt) {
+                (this.session.classificationDoubts ??= new Map()).set(entry.name, entry.doubt);
+            }
         }
         for (const tool of this.upstreamTools) {
             const properties = (tool.inputSchema?.properties ?? {});
@@ -227,6 +243,118 @@ export class ProxyInterceptor {
         this.upstreamTransport = null;
         killSurvivors(descendants);
     }
+    // --effect-probe: take the baseline reading. Called once after connect, before
+    // the first agent call. Throws with a message that says what to fix.
+    async initEffectProbe() {
+        const cfg = this.config.effectProbe;
+        if (!cfg)
+            return;
+        const entry = this.toolClassification.find((t) => t.name === cfg.tool);
+        if (!entry) {
+            throw new Error(`--effect-probe "${cfg.tool}": the upstream has no such tool.`);
+        }
+        if (entry.class !== "read") {
+            throw new Error(`--effect-probe "${cfg.tool}": the probe must be a READ tool, but it is taken for ${entry.class === "write" ? "a write" : "a tool that is neither read nor write"} (${entry.source}). The harness calls it itself, repeatedly: a write would change what it measures. Pick a read tool, or classify it with --read-tools.`);
+        }
+        let reading;
+        try {
+            reading = await this.readProbe();
+        }
+        catch (err) {
+            throw new Error(`--effect-probe "${cfg.tool}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+        this.session.effectProbe = {
+            ...cfg,
+            before: reading.count,
+            after: reading.count,
+            error: null,
+            readAtCalls: 0,
+            ...(reading.warning ? { warning: reading.warning } : {}),
+        };
+    }
+    // The probe is read OFF the answer path: after a call a trailing, debounced
+    // reading is scheduled, and the report file is written again when it lands.
+    // The agent never waits for it, and its timing (retry_after) is not touched.
+    // After an error the next scheduled reading waits 10 s.
+    scheduleEffectProbeRefresh() {
+        const probe = this.session.effectProbe;
+        if (!probe || this.probeTimer || this.closing)
+            return;
+        const wait = probe.error !== null ? 10_000 : 400;
+        this.probeTimer = setTimeout(() => {
+            this.probeTimer = null;
+            void this.refreshEffectProbe().then(() => {
+                this.onProbeRefreshed?.();
+                // Calls were still open (or arrived meanwhile): the reading is behind.
+                if (probe.readAtCalls !== this.session.history.length)
+                    this.scheduleEffectProbeRefresh();
+            });
+        }, wait);
+        this.probeTimer.unref?.();
+    }
+    // Latest reading. `force` (a report is being read, the proxy is ending) waits
+    // for a reading even right after an error. One reading at a time; a second
+    // caller waits for the one in flight and uses it.
+    async refreshEffectProbe(force = false) {
+        const probe = this.session.effectProbe;
+        if (!probe)
+            return;
+        if (this.probeInFlight)
+            await this.probeInFlight;
+        const calls = this.session.history.length;
+        if (calls === probe.readAtCalls && probe.error === null)
+            return;
+        if (!force && probe.error !== null && probe.lastAttemptAt !== undefined && Date.now() - probe.lastAttemptAt < 10_000)
+            return;
+        // Calls still open: a record may be half-written. Leave the old reading;
+        // the report treats open calls as "inconclusive".
+        if ((this.session.pending?.size ?? 0) > 0)
+            return;
+        this.probeInFlight = (async () => {
+            probe.lastAttemptAt = Date.now();
+            try {
+                const reading = await this.readProbe();
+                probe.after = reading.count;
+                probe.error = null;
+                if (reading.warning)
+                    probe.warning = reading.warning;
+                else
+                    delete probe.warning;
+            }
+            catch (err) {
+                probe.error = err instanceof Error ? err.message : String(err);
+            }
+            // The history length BEFORE the reading: calls that arrived during it
+            // make the reading older than the history, and the report says so.
+            probe.readAtCalls = calls;
+        })();
+        try {
+            await this.probeInFlight;
+        }
+        finally {
+            this.probeInFlight = null;
+        }
+    }
+    async readProbe() {
+        const cfg = this.config.effectProbe;
+        if (!cfg)
+            throw new Error("no probe configured");
+        if (!this.upstreamClient || !this.upstreamAlive) {
+            throw new Error("the upstream is not running any more");
+        }
+        let result;
+        try {
+            result = await this.upstreamClient.callTool({ name: cfg.tool, arguments: cfg.args }, undefined, { timeout: PROBE_TIMEOUT_MS });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (/timed out/i.test(message)) {
+                throw new Error(`the probe tool did not answer within ${PROBE_TIMEOUT_MS / 1000} s (use a faster read tool, or narrow it with --effect-probe-args)`);
+            }
+            throw new Error(`the probe call failed: ${message}`);
+        }
+        return parseProbeReading(result, cfg.countPath);
+    }
     getUpstreamTools() {
         return this.upstreamTools;
     }
@@ -255,6 +383,7 @@ export class ProxyInterceptor {
         catch {
             // Reporting must never change what the agent gets back.
         }
+        this.scheduleEffectProbeRefresh();
         return result;
     }
     async handleToolCall(toolName, args, signal) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cuonztech-agent-harness",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "mcpName": "io.github.cuonztech/agent-harness",
   "description": "Chaos-Proxy für MCP-Server: injiziert F1-F5-Fehlerszenarien (Ghost-Write nach Timeout, 429, defektes JSON, Duplicate Dispatch, 500) vor einem echten Upstream-MCP-Server und prüft, ob ein Agent Schreibvorgänge doppelt ausführt. Mit Resilience Score und System-Prompt-Patches für beobachtetes Fehlverhalten.",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/cuonztech/agent-harness",
     "source": "github"
   },
-  "version": "0.8.0",
+  "version": "0.9.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "cuonztech-agent-harness",
-      "version": "0.8.0",
+      "version": "0.9.0",
       "transport": {
         "type": "stdio"
       }