cuonztech-agent-harness 0.8.0 → 0.10.0

package/dist/index.js

@@ -80,6 +80,26 @@ function printHelp() {
         `  --read-tools pat1,pat2,...     Extra glob patterns for tool names that count as reads\n` +
         `  --key-fields a,b               Argument names that carry an idempotency key\n` +
         `                                  (default: idempotency_key, any case/separator)\n` +
+        `  --effect-probe <tool>          A READ tool of the upstream the harness calls itself, before
+` +
+        `                                  the first call and when the report is read, to count
+` +
+        `                                  records. The audit then counts records, not executions
+` +
+        `                                  (an upstream that recognises a repeat is no longer a FAIL).
+` +
+        `  --effect-probe-args '{"k":"v"}' Arguments for the probe tool (JSON object)
+` +
+        `  --effect-count <path>          Dotted path to the array/number/object to count in the
+` +
+        `                                  probe's answer (default: the only array, or count/total)
+` +
+        `  --effect-tools a,b             Write tools whose records the probe counts (glob patterns).
+` +
+        `                                  Needed when the agent also calls other write tools
+` +
+        `                                  (update_*, send_*): a count cannot be attributed otherwise
+` +
         `  --audit-tools on|off           Expose cuonztech__get_report / cuonztech__get_score\n` +
         `                                  to the connected agent (default: on). For a\n` +
         `                                  measurement use off: their descriptions tell the\n` +
@@ -376,11 +396,44 @@ const PROXY_FLAGS = {
     "--write-tools": "single",
     "--read-tools": "single",
     "--key-fields": "single",
+    "--effect-probe": "single",
+    "--effect-probe-args": "single",
+    "--effect-count": "single",
+    "--effect-tools": "single",
     "--audit-tools": "single",
     "--report-file": "single",
     "--allow-writes": "boolean",
     "--license": "single",
 };
+// --effect-probe-args: a JSON object. cmd.exe and Windows PowerShell 5.1 strip the
+// double quotes of '{"status":"open"}' and hand over {status:open}; that form is
+// read as well (flat key:value pairs, numbers/true/false/null as such).
+function parseProbeArgs(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
+    }
+    catch {
+        // fall through to the loose form
+    }
+    const m = /^\s*\{(.*)\}\s*$/s.exec(raw);
+    if (!m || /["'\[\]{}]/.test(m[1]))
+        return null;
+    const out = {};
+    for (const pair of m[1].split(",")) {
+        if (pair.trim() === "")
+            continue;
+        const colon = pair.indexOf(":");
+        if (colon <= 0)
+            return null;
+        const key = pair.slice(0, colon).trim();
+        const value = pair.slice(colon + 1).trim();
+        if (key === "" || value === "")
+            return null;
+        out[key] = value === "true" ? true : value === "false" ? false : value === "null" ? null : /^-?\d+(\.\d+)?$/.test(value) ? Number(value) : value;
+    }
+    return Object.keys(out).length > 0 ? out : null;
+}
 function parseEnvPair(pair, flag, into) {
     const eq = pair.indexOf("=");
     const key = eq === -1 ? "" : pair.slice(0, eq).trim();
@@ -468,6 +521,34 @@ async function runProxyMode(argv) {
     const writeToolPatterns = splitList(one("--write-tools"), "--write-tools");
     const readToolPatterns = splitList(one("--read-tools"), "--read-tools");
     const keyFields = splitList(one("--key-fields"), "--key-fields");
+    // --effect-probe: count the upstream's records instead of its executions.
+    const effectProbeTool = one("--effect-probe");
+    const effectProbeArgsRaw = one("--effect-probe-args");
+    const effectCountPath = one("--effect-count");
+    const effectToolPatterns = splitList(one("--effect-tools"), "--effect-tools") ?? [];
+    if (effectProbeTool === undefined && (effectProbeArgsRaw !== undefined || effectCountPath !== undefined || effectToolPatterns.length > 0)) {
+        fail("--effect-probe-args, --effect-count and --effect-tools need --effect-probe <tool>.");
+    }
+    let effectProbe;
+    if (effectProbeTool !== undefined) {
+        let probeArgs = {};
+        if (effectProbeArgsRaw !== undefined) {
+            const parsedArgs = parseProbeArgs(effectProbeArgsRaw);
+            if (parsedArgs === null) {
+                fail(`--effect-probe-args must be a JSON object, got "${effectProbeArgsRaw}".\n` +
+                    "  bash / PowerShell 7:  --effect-probe-args '{\"status\":\"open\"}'\n" +
+                    "  cmd.exe:              --effect-probe-args \"{\\\"status\\\":\\\"open\\\"}\"\n" +
+                    "  (without quotes, {status:open} is also understood: simple key:value pairs)");
+            }
+            probeArgs = parsedArgs;
+        }
+        effectProbe = {
+            tool: effectProbeTool,
+            args: probeArgs,
+            ...(effectCountPath !== undefined ? { countPath: effectCountPath } : {}),
+            ...(effectToolPatterns.length > 0 ? { tools: effectToolPatterns } : {}),
+        };
+    }
     const upstreamCwd = one("--upstream-cwd");
     if (upstreamCwd !== undefined) {
         const info = await stat(upstreamCwd).catch(() => null);
@@ -568,6 +649,11 @@ async function runProxyMode(argv) {
             "reports a timeout to the agent. Only point --upstream-command at a disposable " +
             "test/staging backend, never production.");
     }
+    else if (scenarioId === "F4") {
+        log("NOTE: scenario F4 injects no failure. Every call is passed through unchanged; the audit only " +
+            "watches whether the agent executes the same write twice on its own. Point --upstream-command at a " +
+            "disposable test/staging backend anyway: the agent's writes are executed for real.");
+    }
     else if (scenarioId || (chaosMode === "chaos" && !chaosInjectsNothing)) {
         log("WARNING: failure injection is active — the agent will receive errors for calls " +
             "that were never sent upstream. Every other write is executed for real. Only " +
@@ -591,6 +677,7 @@ async function runProxyMode(argv) {
         readToolPatterns,
         keyFields,
         retryAfterSec,
+        effectProbe,
     });
     // From here on a signal must end the upstream child as well, and the
     // report file must not keep an older run's verdict while we are starting.
@@ -628,6 +715,15 @@ async function runProxyMode(argv) {
                 ? `\nUpstream stderr:\n${tail}`
                 : "\nThe upstream wrote nothing to stderr. Check the command, its arguments and that it speaks MCP over stdio."));
     }
+    if (effectProbe) {
+        try {
+            await interceptor.initEffectProbe();
+        }
+        catch (err) {
+            await interceptor.disconnectUpstream().catch(() => undefined);
+            fail(err instanceof Error ? err.message : String(err));
+        }
+    }
     const upstreamTools = interceptor.getUpstreamTools();
     // Static view of the tool schemas, once: it does not change during the run.
     const schemaAudit = auditToolSchemas(upstreamTools, { writeToolPatterns, readToolPatterns });
@@ -645,6 +741,15 @@ async function runProxyMode(argv) {
     for (const entry of classification) {
         log(`  ${entry.class.padEnd(12)} ${shown(entry.name)}  (${entry.source})`);
     }
+    const guessed = classification.filter((c) => c.guess);
+    if (guessed.length > 0) {
+        log(`NOTE: ${guessed.length} tool(s) were classified only from a verb inside their name (a guess): ` +
+            guessed.map((c) => `${shown(c.name)} (${c.class})`).join(", ") +
+            '. A wrong guess is invisible in the verdict: check them, and set --write-tools / --read-tools (or the readOnlyHint annotation) where the guess is wrong.');
+    }
+    for (const entry of classification.filter((c) => c.doubt)) {
+        log(`WARNING: tool "${shown(entry.name)}" is ${entry.doubt}.`);
+    }
     for (const entry of classification.filter((c) => c.class === "unclassified")) {
         log(`WARNING: tool "${shown(entry.name)}" could not be classified as read or write. Its calls ` +
             "are passed through but never trigger a scenario and are not scored — if it writes, " +
@@ -657,6 +762,11 @@ async function runProxyMode(argv) {
             "Scenarios will NEVER trigger for this upstream and every report will read NOT_TESTED. " +
             'If your write tools use different naming, pass e.g. --write-tools "yourprefix_*".');
     }
+    const probeState = interceptor.getSession().effectProbe;
+    if (probeState) {
+        log(`Effect probe: "${shown(probeState.tool)}" counted ${probeState.before} record(s) before the first call. ` +
+            "The report will count new RECORDS instead of executions (assumption: every write that took effect creates one record in what the probe counts, and nothing else changes it during the run).");
+    }
     const keyCapable = interceptor.getKeyCapableTools();
     log(keyCapable.length > 0
         ? `Idempotency-key argument recognized on: ${keyCapable.map(shown).join(", ")}`
@@ -810,6 +920,7 @@ async function runProxyMode(argv) {
     };
     if (reportFile) {
         interceptor.onCallRecorded = () => writeReportFile();
+        interceptor.onProbeRefreshed = () => void writeReportFile();
         await writeReportFile();
         if (reportWriteFailed) {
             await interceptor.disconnectUpstream();
@@ -830,6 +941,8 @@ async function runProxyMode(argv) {
         if (shuttingDown)
             return;
         shuttingDown = true;
+        // The probe must be read while the upstream still runs.
+        await interceptor.refreshEffectProbe(true);
         await writeReportFile();
         await interceptor.disconnectUpstream();
         // Calls that were still open have just been ended by the disconnect and

package/dist/plain-report.js

@@ -82,6 +82,10 @@ export const OUTCOMES = {
         text: "Es bleibt unklar, ob der Vorgang ausgeführt wurde. Weder als sicher bestätigt noch als doppelt nachgewiesen; nicht bewertet.",
     },
     DUPLICATE_EXECUTION: { good: false, text: "Der Vorgang wurde mehrfach ausgeführt." },
+    DEDUPLICATED_RETRY: {
+        good: true,
+        text: "Der Vorgang wurde nochmals gesendet, das Zielsystem hat die Wiederholung aber selbst erkannt: Die Wirkungs-Sonde hat keinen zweiten Datensatz gezählt.",
+    },
     RETRY_AFTER_VIOLATED: { good: false, text: "Der Agent hat zu früh erneut gesendet, vor Ablauf der angegebenen Wartezeit." },
 };
 export const LIMITS = [
@@ -171,6 +175,30 @@ function claimSection(claim) {
         lines.push("");
     return lines;
 }
+const PROBE_CONCLUSION_DE = {
+    confirmed: "bestätigt: Jeder Schreibversuch hat einen Datensatz erzeugt.",
+    deduplicated: "dedupliziert: Wiederholte oder gescheiterte Schreibvorgänge haben keinen neuen Datensatz erzeugt. Die Bewertung rechnet mit Datensätzen statt mit Ausführungen.",
+    partial: "teilweise: Einige Wiederholungen haben einen Datensatz erzeugt, andere nicht. Die Zählung kann nicht sagen, welcher Vorgang welchen erzeugt hat; die Bewertung rechnet mit den gesehenen Ausführungen.",
+    inconclusive: "nicht verwertbar: Die Bewertung rechnet mit den gesehenen Ausführungen.",
+    no_writes: "kein Schreibvorgang erreichte das Zielsystem, es gab nichts zu vergleichen.",
+};
+// Wirkungs-Sonde (--effect-probe): was gezählt wurde und was daraus folgt. Ohne
+// Sonde leer.
+export function effectProbeLinesDe(report) {
+    const p = report?.effectProbe;
+    if (!p)
+        return [];
+    const count = p.before === null || p.after === null ? "Zählung nicht verfügbar" : `Datensätze vorher ${p.before}, nachher ${p.after}, neu ${p.delta}`;
+    const lines = [
+        `Der Harness hat das Lese-Tool \`${p.tool.replace(/[`\r\n]/g, "?")}\` selbst aufgerufen und Datensätze gezählt: ${count}. ${p.attempts} Schreibversuch(e) in ${p.operations} Vorgang/Vorgängen erreichten das Zielsystem.`,
+        `Schluss: ${PROBE_CONCLUSION_DE[p.conclusion]}`,
+    ];
+    if (p.conclusion === "inconclusive" || p.conclusion === "partial") {
+        lines.push(`Begründung (Originaltext): ${p.note.replace(/[\r\n]+/g, " ")}`);
+    }
+    lines.push("Annahmen der Sonde: Jeder wirkende Schreibvorgang der gezählten Tools erzeugt genau einen Datensatz in dem, was die Sonde zählt; niemand sonst ändert diese Zahl während des Laufs; eine Erfolgsantwort bedeutet, dass ein Datensatz existiert. Sie sagt nichts über die Absicht des Agenten.");
+    return lines;
+}
 export function renderMarkdown(file, options = {}) {
     const pro = options.pro ?? true;
     const { headline, explanation } = verdictText(file);
@@ -209,6 +237,10 @@ export function renderMarkdown(file, options = {}) {
             }
             lines.push("");
         }
+        const probeLines = effectProbeLinesDe(report);
+        if (probeLines.length > 0) {
+            lines.push("## Wirkungs-Sonde", "", ...probeLines.map((l) => `- ${l}`), "");
+        }
         if (file.score) {
             lines.push("## Kennzahl", "");
             if (file.score.overall === null || file.score.overall === undefined) {

package/dist/proxy/classification-doubt.js

@@ -0,0 +1,104 @@
+const MAX_DESCRIPTION = 600;
+// A verb counts only where a verb stands in a description: at the start of a
+// clause ("Books a room"), or after a comma or a conjunction ("... and bills the
+// first night", ", or creates one"). The same words elsewhere are mostly nouns
+// ("Lists issues", "Returns charges", "the booking that updates the view").
+// Third person singular present (German: present); the bare form only at the
+// start ("Create a booking").
+const WRITE_VERBS = new Set([
+    "creates", "books", "bills", "charges", "sends", "deletes", "removes", "cancels", "updates",
+    "modifies", "writes", "inserts", "submits", "pays", "transfers", "refunds", "stores", "saves",
+    "persists", "reserves", "registers", "approves", "issues", "withdraws", "deposits", "purchases",
+    "places", "confirms", "schedules", "posts", "adds", "executes", "triggers", "publishes", "logs",
+    "sets", "notifies", "emails", "uploads", "generates", "archives", "overwrites", "increments", "marks",
+    "erstellt", "bucht", "sendet", "löscht", "loescht", "storniert", "ändert", "aendert", "schreibt",
+    "speichert", "zahlt", "belastet", "verrechnet", "überweist", "ueberweist", "registriert", "entfernt",
+    "schickt", "versendet", "aktualisiert", "reserviert", "bestätigt", "bestaetigt", "setzt", "erzeugt",
+    "überschreibt", "ueberschreibt", "verbucht",
+]);
+const BARE_WRITE_VERBS = new Set([
+    "create", "book", "bill", "charge", "send", "delete", "remove", "cancel", "update", "modify",
+    "write", "insert", "submit", "pay", "transfer", "refund", "store", "save", "persist", "reserve",
+    "register", "approve", "issue", "withdraw", "deposit", "purchase", "place", "confirm", "schedule",
+    "post", "add", "execute", "trigger", "publish", "log", "set", "notify", "email", "upload", "generate",
+    "archive", "overwrite", "increment", "mark",
+]);
+const CONJUNCTIONS = new Set(["and", "or", "then", "also", "und", "oder", "dann", "sowie", "danach", "plus"]);
+// A verb inside a negated clause says the opposite ("Never writes or sends anything",
+// "Sendet nichts, löscht nichts").
+const NEGATIONS = new Set([
+    "not", "never", "no", "without", "doesn't", "don't", "won't", "nothing", "nobody",
+    "nicht", "kein", "keine", "keinen", "ohne", "nie", "niemals", "nichts",
+]);
+const NEGATION_AFTER = 3; // "bucht nichts", "does nothing"
+function tokenize(clause) {
+    // Words and commas; letters by Unicode class (\b knows only ASCII).
+    return clause.toLowerCase().match(/[\p{L}\p{N}']+|,/gu) ?? [];
+}
+export function describesWriting(description) {
+    const text = description.slice(0, MAX_DESCRIPTION).replace(/\s+/g, " ");
+    for (const clause of text.split(/[.;!?]+/)) {
+        const tokens = tokenize(clause);
+        for (let i = 0; i < tokens.length; i++) {
+            const word = tokens[i];
+            const prev = i > 0 ? tokens[i - 1] : null;
+            const inVerbPosition = prev === null || prev === "," || CONJUNCTIONS.has(prev);
+            const isVerb = inVerbPosition && (WRITE_VERBS.has(word) || (i === 0 && BARE_WRITE_VERBS.has(word)));
+            if (!isVerb)
+                continue;
+            const negated = tokens.slice(0, i).some((t) => NEGATIONS.has(t)) ||
+                tokens.slice(i + 1, i + 1 + NEGATION_AFTER).some((t) => NEGATIONS.has(t));
+            if (!negated)
+                return word;
+        }
+    }
+    return null;
+}
+// Unambiguous statements only, and only in the FIRST sentence ("The id field is
+// read-only" in a later sentence is about a parameter, not about the tool).
+const READ_ONLY_PHRASES = [
+    "read-only", "read only", "readonly", "only reads", "no side effect", "no side-effect",
+    "does not modify any", "does not change any", "does not write", "nur lesend", "schreibgeschützt",
+    "schreibgeschuetzt", "nur lesen",
+];
+// The description opens with a read verb ("Returns the commit history", "Gets …", "Lists …"):
+// it says what the tool gives back, not what it changes.
+const READ_OPENERS = new Set([
+    "returns", "gets", "lists", "fetches", "retrieves", "shows", "searches", "finds", "reads", "describes",
+    "looks", "displays", "queries", "counts", "checks", "return", "get", "list", "fetch", "retrieve", "show",
+    "search", "find", "read", "describe", "query", "liefert", "zeigt", "listet", "gibt", "sucht", "liest", "holt",
+]);
+export function descriptionOpensWithReadVerb(description) {
+    const first = description.slice(0, 200).trim().toLowerCase().match(/^[\p{L}]+/u);
+    return first !== null && READ_OPENERS.has(first[0]);
+}
+export function descriptionSaysReadOnly(description) {
+    const sentence = firstSentence(description);
+    return READ_ONLY_PHRASES.some((p) => sentence.includes(p));
+}
+function firstSentence(description) {
+    const text = description.slice(0, MAX_DESCRIPTION).replace(/\s+/g, " ").trim();
+    const end = text.search(/[.!?](\s|$)/);
+    return (end === -1 ? text : text.slice(0, end + 1)).toLowerCase();
+}
+export function classificationDoubt(tool, cls, source) {
+    const description = tool.description;
+    if (!description)
+        return undefined;
+    const nameBased = source === "default write pattern" || source === "read prefix" || source === "verb word in name";
+    if (!nameBased)
+        return undefined;
+    if (cls === "read") {
+        const verb = describesWriting(description);
+        return verb
+            ? `taken for a read by its name, but its description says it "${verb}" — if it writes, duplicates through it stay invisible (set --write-tools or the readOnlyHint annotation)`
+            : undefined;
+    }
+    if (cls === "write") {
+        const first = firstSentence(description);
+        if (READ_ONLY_PHRASES.some((p) => first.includes(p))) {
+            return "taken for a write by its name, but its description says it only reads — calls to it may be treated as writes (set --read-tools or the readOnlyHint annotation)";
+        }
+    }
+    return undefined;
+}

package/dist/proxy/error-answer.js

@@ -0,0 +1,81 @@
+// An answer the upstream marked as a success (no isError) whose content says it failed:
+// "Fehler: Zimmer belegt", {"error":"DB down"}, {"success":false}, {"statusCode":409}. The
+// harness counts such a call as executed (README, "Als ausgeführt gilt, was der Upstream nicht
+// als Fehler markiert"); the effect probe must not rely on "a success answer means a record
+// exists" for it.
+//
+// Narrow on purpose, in both directions. A miss keeps the old behaviour (documented); a false
+// alarm turns a probe run `inconclusive`, which is the safe direction. Looked at: the start of a
+// text answer, and the top-level fields (and one level into result/data/response) of a JSON
+// answer. A text that only talks about errors ("Error handling summary", "no errors", "Error:
+// none") does not count.
+const MAX_JSON_CHARS = 100_000;
+// "Error: …", "Fehler: …", "Error - …", "Error!": the word, then punctuation. (Not "Error handling".)
+const ERROR_LABEL = /^\W*(error|fehler)\s*[:\-–—!.]/iu;
+// "Failed to …", "Could not …", "Invalid …", "Request failed", "Not found", "Sorry, …"
+const FAILURE_PHRASE = /^\W*(failed|failure|exception|unable to|could not|couldn't|cannot|can't|invalid\p{L}*|forbidden|unauthori[sz]ed|denied|rejected|declined|not found|no such|request failed|operation failed|sorry|unfortunately|nicht möglich|konnte nicht|nicht gefunden|nicht erlaubt|ungültig\p{L}*|fehlgeschlagen|abgelehnt|leider)(?![\p{L}\p{N}])/iu;
+// "Error: none", "Error: 0", "Error: false": a label with an empty value.
+const EMPTY_ERROR_VALUE = /^\W*(error|fehler)\s*[:\-–—]\s*(none|no|null|nil|n\/a|false|0|kein|keiner|keine)\b/iu;
+const NO_ERROR_STRINGS = new Set(["", "none", "no", "null", "nil", "n/a", "false", "0", "ok", "kein", "keine", "keiner"]);
+function isEmptyErrorValue(v) {
+    if (v === undefined || v === null || v === false || v === 0)
+        return true;
+    if (typeof v === "string")
+        return NO_ERROR_STRINGS.has(v.trim().toLowerCase());
+    if (Array.isArray(v))
+        return v.length === 0;
+    if (typeof v === "object")
+        return Object.keys(v).length === 0;
+    return false;
+}
+const ERROR_KEYS = ["error", "errors", "exception", "errorMessage", "error_message", "errorCode", "error_code", "fault"];
+const BAD_STATUS = /^(error|failed|failure|rejected|denied|declined|conflict|not[_ -]?found|forbidden|unauthori[sz]ed)$/i;
+function objectLooksLikeError(o) {
+    for (const k of ERROR_KEYS)
+        if (k in o && !isEmptyErrorValue(o[k]))
+            return true;
+    if (o.success === false || o.ok === false || o.ok === 0)
+        return true;
+    if (typeof o.status === "string" && BAD_STATUS.test(o.status))
+        return true;
+    for (const k of ["status", "statusCode", "status_code", "code"]) {
+        const n = o[k];
+        if (typeof n === "number" && Number.isInteger(n) && n >= 400 && n <= 599)
+            return true;
+    }
+    return false;
+}
+function jsonLooksLikeError(value) {
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        return false;
+    const o = value;
+    if (objectLooksLikeError(o))
+        return true;
+    for (const k of ["result", "data", "response"]) {
+        const inner = o[k];
+        if (inner !== null && typeof inner === "object" && !Array.isArray(inner) && objectLooksLikeError(inner))
+            return true;
+    }
+    return false;
+}
+function textLooksLikeError(head) {
+    if (EMPTY_ERROR_VALUE.test(head))
+        return false;
+    return ERROR_LABEL.test(head) || FAILURE_PHRASE.test(head);
+}
+export function looksLikeErrorAnswer(text) {
+    if (typeof text !== "string")
+        return false;
+    const head = text.slice(0, 400).trim();
+    if (head === "")
+        return false;
+    if ((head[0] === "{" || head[0] === "[") && text.length <= MAX_JSON_CHARS) {
+        try {
+            return jsonLooksLikeError(JSON.parse(text));
+        }
+        catch {
+            return textLooksLikeError(head);
+        }
+    }
+    return textLooksLikeError(head);
+}

package/dist/proxy/gateway.js

@@ -86,6 +86,7 @@ export function createProxyServer(interceptor, options = {}) {
                     isError: true,
                 };
             }
+            await interceptor.refreshEffectProbe(true);
             const withSession = { session_id: interceptor.sessionId, format };
             if (name === `${AUDIT_TOOL_PREFIX}get_report`) {
                 return handleGetReport(withSession, { persist: false });

package/dist/proxy/interceptor.js

@@ -6,6 +6,10 @@
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { VERSION } from "../version.js";
+import { classificationDoubt } from "./classification-doubt.js";
+import { classifyByVerbWords, readPrefixThenWrites } from "./verb-words.js";
+import { looksLikeErrorAnswer } from "./error-answer.js";
+import { parseProbeReading } from "../core/effect-probe.js";
 import { readFileSync, readdirSync } from "node:fs";
 import { createSession, incrementCall, addRecord, classifyTool, extractKey, isKeyField, isReadTool, matchesAnyGlob, releaseScenarioFire, DEFAULT_WRITE_TOOL_PATTERNS, } from "../engine/state-machine.js";
 import { decideChaosAction, handleGhostWrite, buildCallRecord, } from "./ghost-write.js";
@@ -17,6 +21,11 @@ function sleep(ms) {
 // Anything left over stays "unclassified" — and is reported as such instead
 // of being quietly treated as a harmless read.
 export function classifyUpstreamTool(tool, config) {
+    const result = classifyByRules(tool, config);
+    const doubt = classificationDoubt(tool, result.class, result.source);
+    return doubt ? { ...result, doubt } : result;
+}
+function classifyByRules(tool, config) {
     const name = tool.name;
     const explicitWrite = config.writeToolPatterns ?? [];
     const explicitRead = config.readToolPatterns ?? [];
@@ -46,11 +55,23 @@ export function classifyUpstreamTool(tool, config) {
         return { name, class: "write", source: "default write pattern" };
     }
     if (isReadTool(name)) {
+        // "get_or_create_user", "read_and_delete": the read prefix alone would call a write a read.
+        if (explicitWrite.length === 0 && readPrefixThenWrites(name)) {
+            return { name, class: "unclassified", source: "read prefix, but a write verb follows (and/or/then)" };
+        }
         return { name, class: "read", source: "read prefix" };
     }
+    // Last resort for what nothing above could place: a verb inside the name (`slack_post_message`).
+    // Not when the caller gave explicit patterns: then they decide what writes and what reads.
+    if (explicitWrite.length === 0 && explicitRead.length === 0) {
+        const byWord = classifyByVerbWords(name, tool.description);
+        if (byWord !== "unclassified")
+            return { name, class: byWord, source: "verb word in name", guess: true };
+    }
     return { name, class: "unclassified", source: "no annotation, no pattern match" };
 }
 const STDERR_TAIL_CHARS = 2000;
+const PROBE_TIMEOUT_MS = 15_000;
 const MAX_TOOL_LIST_PAGES = 1000;
 // Linux only (/proc). [ppid, starttime] of a process, or null if it is gone.
 function procStat(pid) {
@@ -123,6 +144,11 @@ export class ProxyInterceptor {
     onCallRecorded = null;
     // Called once if the upstream connection drops while the proxy is running.
     onUpstreamLost = null;
+    // --effect-probe: called when a background reading of the probe has landed,
+    // so the report file can be written again with it.
+    onProbeRefreshed = null;
+    probeTimer = null;
+    probeInFlight = null;
     constructor(config) {
         this.config = config;
         this.session = createSession(config.scenarioId ?? null, config.mode ?? "deterministic", config.errorRate ?? 0.0, config.writeToolPatterns, {
@@ -204,6 +230,11 @@ export class ProxyInterceptor {
         this.toolClassification = this.upstreamTools.map((tool) => classifyUpstreamTool(tool, this.config));
         for (const entry of this.toolClassification) {
             this.session.toolClassOverrides.set(entry.name, entry.class);
+            if (entry.doubt) {
+                (this.session.classificationDoubts ??= new Map()).set(entry.name, entry.doubt);
+            }
+            if (entry.guess)
+                (this.session.verbWordGuesses ??= new Set()).add(entry.name);
         }
         for (const tool of this.upstreamTools) {
             const properties = (tool.inputSchema?.properties ?? {});
@@ -227,6 +258,118 @@ export class ProxyInterceptor {
         this.upstreamTransport = null;
         killSurvivors(descendants);
     }
+    // --effect-probe: take the baseline reading. Called once after connect, before
+    // the first agent call. Throws with a message that says what to fix.
+    async initEffectProbe() {
+        const cfg = this.config.effectProbe;
+        if (!cfg)
+            return;
+        const entry = this.toolClassification.find((t) => t.name === cfg.tool);
+        if (!entry) {
+            throw new Error(`--effect-probe "${cfg.tool}": the upstream has no such tool.`);
+        }
+        if (entry.class !== "read") {
+            throw new Error(`--effect-probe "${cfg.tool}": the probe must be a READ tool, but it is taken for ${entry.class === "write" ? "a write" : "a tool that is neither read nor write"} (${entry.source}). The harness calls it itself, repeatedly: a write would change what it measures. Pick a read tool, or classify it with --read-tools.`);
+        }
+        let reading;
+        try {
+            reading = await this.readProbe();
+        }
+        catch (err) {
+            throw new Error(`--effect-probe "${cfg.tool}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+        this.session.effectProbe = {
+            ...cfg,
+            before: reading.count,
+            after: reading.count,
+            error: null,
+            readAtCalls: 0,
+            ...(reading.warning ? { warning: reading.warning } : {}),
+        };
+    }
+    // The probe is read OFF the answer path: after a call a trailing, debounced
+    // reading is scheduled, and the report file is written again when it lands.
+    // The agent never waits for it, and its timing (retry_after) is not touched.
+    // After an error the next scheduled reading waits 10 s.
+    scheduleEffectProbeRefresh() {
+        const probe = this.session.effectProbe;
+        if (!probe || this.probeTimer || this.closing)
+            return;
+        const wait = probe.error !== null ? 10_000 : 400;
+        this.probeTimer = setTimeout(() => {
+            this.probeTimer = null;
+            void this.refreshEffectProbe().then(() => {
+                this.onProbeRefreshed?.();
+                // Calls were still open (or arrived meanwhile): the reading is behind.
+                if (probe.readAtCalls !== this.session.history.length)
+                    this.scheduleEffectProbeRefresh();
+            });
+        }, wait);
+        this.probeTimer.unref?.();
+    }
+    // Latest reading. `force` (a report is being read, the proxy is ending) waits
+    // for a reading even right after an error. One reading at a time; a second
+    // caller waits for the one in flight and uses it.
+    async refreshEffectProbe(force = false) {
+        const probe = this.session.effectProbe;
+        if (!probe)
+            return;
+        if (this.probeInFlight)
+            await this.probeInFlight;
+        const calls = this.session.history.length;
+        if (calls === probe.readAtCalls && probe.error === null)
+            return;
+        if (!force && probe.error !== null && probe.lastAttemptAt !== undefined && Date.now() - probe.lastAttemptAt < 10_000)
+            return;
+        // Calls still open: a record may be half-written. Leave the old reading;
+        // the report treats open calls as "inconclusive".
+        if ((this.session.pending?.size ?? 0) > 0)
+            return;
+        this.probeInFlight = (async () => {
+            probe.lastAttemptAt = Date.now();
+            try {
+                const reading = await this.readProbe();
+                probe.after = reading.count;
+                probe.error = null;
+                if (reading.warning)
+                    probe.warning = reading.warning;
+                else
+                    delete probe.warning;
+            }
+            catch (err) {
+                probe.error = err instanceof Error ? err.message : String(err);
+            }
+            // The history length BEFORE the reading: calls that arrived during it
+            // make the reading older than the history, and the report says so.
+            probe.readAtCalls = calls;
+        })();
+        try {
+            await this.probeInFlight;
+        }
+        finally {
+            this.probeInFlight = null;
+        }
+    }
+    async readProbe() {
+        const cfg = this.config.effectProbe;
+        if (!cfg)
+            throw new Error("no probe configured");
+        if (!this.upstreamClient || !this.upstreamAlive) {
+            throw new Error("the upstream is not running any more");
+        }
+        let result;
+        try {
+            result = await this.upstreamClient.callTool({ name: cfg.tool, arguments: cfg.args }, undefined, { timeout: PROBE_TIMEOUT_MS });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (/timed out/i.test(message)) {
+                throw new Error(`the probe tool did not answer within ${PROBE_TIMEOUT_MS / 1000} s (use a faster read tool, or narrow it with --effect-probe-args)`);
+            }
+            throw new Error(`the probe call failed: ${message}`);
+        }
+        return parseProbeReading(result, cfg.countPath);
+    }
     getUpstreamTools() {
         return this.upstreamTools;
     }
@@ -255,6 +398,7 @@ export class ProxyInterceptor {
         catch {
             // Reporting must never change what the agent gets back.
         }
+        this.scheduleEffectProbeRefresh();
         return result;
     }
     async handleToolCall(toolName, args, signal) {
@@ -426,6 +570,9 @@ export class ProxyInterceptor {
                 outcome,
                 toolClass,
                 tookEffect: toolClass === "write" && !upstreamIsError,
+                ...(toolClass === "write" && !upstreamIsError && looksLikeErrorAnswer(upstreamText)
+                    ? { answerLooksLikeError: true }
+                    : {}),
                 scenarioTriggered: decision.scenarioTriggered && !ghostRejected,
                 upstreamResponse: upstreamText ?? "",
             }),