cuonztech-agent-harness 0.3.2 → 0.5.0

package/dist/core/state-engine.js

@@ -1,3 +1,2 @@
 // Re-export from new engine module for backward compatibility
-export { createSession, getSession, incrementCall, addRecord, resetSession, deleteSession, listSessions, isReadTool, isWriteTool, markGhostCommitted, } from "../engine/state-machine.js";
-//# sourceMappingURL=state-engine.js.map
+export { createSession, getSession, incrementCall, addRecord, resetSession, deleteSession, listSessions, isReadTool, isWriteTool, classifyTool, extractKey, stripKeyFields, reserveScenarioFire, releaseScenarioFire, markGhostCommitted, } from "../engine/state-machine.js";

package/dist/engine/state-diff.js

@@ -42,7 +42,10 @@ export class StateDiffStore {
             recordExisted: record !== undefined,
             valueBeforeRetry: record?.value ?? null,
             valueAfterRetry: record?.value ?? null,
-            wasOverwritten: record?.overwrittenAt !== null,
+            // A key that was never written cannot have been overwritten:
+            // `record?.overwrittenAt !== null` alone is true for a missing record
+            // (undefined !== null).
+            wasOverwritten: record !== undefined && record.overwrittenAt !== null,
             agentCheckedBeforeRetry: agentRead,
             agentGaveUpSilently: !agentRead && !agentHadWarning,
         };
@@ -52,4 +55,3 @@ export class StateDiffStore {
         this.readLog = [];
     }
 }
-//# sourceMappingURL=state-diff.js.map

package/dist/engine/state-machine.js

@@ -1,7 +1,48 @@
 import { randomUUID } from "node:crypto";
 import { StateDiffStore } from "./state-diff.js";
 const sessions = new Map();
-export function createSession(scenarioId = null, mode = "deterministic", errorRate = 0.0) {
+// Upper bound on sessions kept in memory: a long-running server whose
+// clients never call delete_session would otherwise grow without limit.
+// The oldest session is dropped first.
+const MAX_SESSIONS = 500;
+// Heuristic only: a list of common write-verb prefixes can never be
+// complete. Tools that match nothing here (and no read prefix) are reported
+// as "unclassified" rather than silently treated as harmless — see
+// classifyTool(). Callers with unusual naming pass their own patterns
+// (--write-tools in proxy mode, start_session's write_tool_patterns); in
+// proxy mode MCP tool annotations (readOnlyHint) take precedence over this
+// list.
+export const DEFAULT_WRITE_TOOL_PATTERNS = [
+    "write*",
+    "create_*",
+    "send_*",
+    "submit_*",
+    "post_*",
+    "update_*",
+    "insert_*",
+    "book_*",
+    "pay_*",
+    "charge_*",
+    "cancel_*",
+    "delete_*",
+    "confirm_*",
+    "place_*",
+    "add_*",
+    "register_*",
+    "schedule_*",
+    "transfer_*",
+    "refund_*",
+    "reserve_*",
+    "issue_*",
+    "remove_*",
+    "withdraw_*",
+    "deposit_*",
+    "purchase_*",
+    "approve_*",
+];
+export const DEFAULT_KEY_FIELDS = ["idempotency_key"];
+export const DEFAULT_RETRY_AFTER_SEC = 60;
+export function createSession(scenarioId = null, mode = "deterministic", errorRate = 0.0, writeToolPatterns = DEFAULT_WRITE_TOOL_PATTERNS, options = {}) {
     const session = {
         sessionId: randomUUID(),
         callCount: 0,
@@ -12,8 +53,30 @@ export function createSession(scenarioId = null, mode = "deterministic", errorRa
         keyStates: new Map(),
         createdAt: Date.now(),
         stateDiff: new StateDiffStore(),
+        writeToolPatterns: writeToolPatterns.length > 0 ? writeToolPatterns : DEFAULT_WRITE_TOOL_PATTERNS,
+        readToolPatterns: options.readToolPatterns ?? [],
+        toolClassOverrides: new Map(),
+        keyFields: options.keyFields && options.keyFields.length > 0
+            ? options.keyFields
+            : DEFAULT_KEY_FIELDS,
+        keyCapableTools: new Set(),
+        retryAfterSec: options.retryAfterSec ?? DEFAULT_RETRY_AFTER_SEC,
+        scenarioFireCount: 0,
+        knownToolsOnly: false,
+        kind: "demo",
     };
     sessions.set(session.sessionId, session);
+    if (sessions.size > MAX_SESSIONS) {
+        let oldest = null;
+        for (const s of sessions.values()) {
+            if (s.sessionId === session.sessionId)
+                continue;
+            if (!oldest || s.createdAt < oldest.createdAt)
+                oldest = s;
+        }
+        if (oldest)
+            sessions.delete(oldest.sessionId);
+    }
     return session;
 }
 export function getSession(sessionId) {
@@ -26,30 +89,86 @@ export function incrementCall(sessionId) {
     session.callCount += 1;
     return session.callCount;
 }
+function normalizeFieldName(name) {
+    return name.toLowerCase().replace(/[-_]/g, "");
+}
+export function isKeyField(name, keyFields) {
+    const normalized = normalizeFieldName(name);
+    return keyFields.some((f) => normalizeFieldName(f) === normalized);
+}
+// The idempotency key a call carries, if any. An empty string counts as
+// "no key".
+export function extractKey(args, keyFields = DEFAULT_KEY_FIELDS) {
+    for (const [name, value] of Object.entries(args)) {
+        if (!isKeyField(name, keyFields))
+            continue;
+        if (typeof value === "string" && value.length > 0)
+            return value;
+        if (typeof value === "number")
+            return String(value);
+    }
+    return undefined;
+}
+export function stripKeyFields(args, keyFields = DEFAULT_KEY_FIELDS) {
+    const rest = {};
+    for (const [name, value] of Object.entries(args)) {
+        if (!isKeyField(name, keyFields))
+            rest[name] = value;
+    }
+    return rest;
+}
+export function classifyTool(session, toolName) {
+    const override = session.toolClassOverrides.get(toolName);
+    if (override)
+        return override;
+    if (session.knownToolsOnly)
+        return "unclassified";
+    if (matchesAnyGlob(toolName, session.writeToolPatterns))
+        return "write";
+    if (isReadTool(toolName) || matchesAnyGlob(toolName, session.readToolPatterns)) {
+        return "read";
+    }
+    return "unclassified";
+}
+// Marks that the scenario fired on one more write. Deliberately synchronous:
+// the caller must not await anything between receiving a call and deciding,
+// or two parallel writes would both see "the scenario has not fired yet".
+export function reserveScenarioFire(session) {
+    session.scenarioFireCount += 1;
+}
+// Gives a reserved fire back: the scenario was meant for this call but could
+// not be applied (the upstream call failed or was rejected), so the next
+// write gets it instead.
+export function releaseScenarioFire(session) {
+    session.scenarioFireCount = Math.max(0, session.scenarioFireCount - 1);
+}
 export function addRecord(sessionId, record) {
     const session = sessions.get(sessionId);
     if (!session)
         throw new Error(`Session ${sessionId} not found`);
+    record.upstreamExecuted = Boolean(record.upstreamExecuted);
+    record.outcome ??= record.injectedError !== null ? "injected_failure" : "success";
+    record.toolClass ??= classifyTool(session, record.toolName);
+    record.tookEffect ??=
+        !record.deduplicated &&
+            record.toolClass === "write" &&
+            (record.outcome === "success" ||
+                (record.outcome === "injected_failure" && record.upstreamExecuted));
     session.history.push(record);
     // Cache replay of an already-committed key: stays in history for the call
     // count, but is a no-op for key-tracking — nothing new was written.
     if (record.deduplicated)
         return;
-    const key = record.args["idempotency_key"];
+    const key = extractKey(record.args, session.keyFields);
     if (!key)
         return;
-    const isWrite = isWriteTool(record.toolName);
-    const hasInjectedError = record.injectedError !== null && !record.injectedError.includes("SUCCESS");
-    const hasResponseError = typeof record.response === "string" && record.response.includes('"error"');
-    const isError = hasInjectedError || hasResponseError;
-    // Real-value tracking, independent of the keyStates classification below.
-    // A write only actually lands in the backing store if it wasn't rejected
-    // outright — either it succeeded cleanly, or (ghost-write) the upstream
-    // executed it despite the agent seeing an error.
-    if (isWrite && (!isError || record.upstreamExecuted)) {
+    const isWrite = record.toolClass === "write";
+    const isError = record.outcome !== "success";
+    // Real-value tracking, independent of the keyStates bookkeeping below.
+    if (isWrite && record.tookEffect) {
         session.stateDiff.write(key, record.response, record.callNumber);
     }
-    else if (isReadTool(record.toolName)) {
+    else if (record.toolClass === "read") {
         session.stateDiff.read(key, record.callNumber);
     }
     const existing = session.keyStates.get(key);
@@ -78,7 +197,7 @@ export function addRecord(sessionId, record) {
         if ((existing.state === "FAILED_DOWNSTREAM" || existing.state === "GHOST_COMMITTED") && !isError) {
             const hasReadBetween = session.history.some((r) => r.callNumber > existing.firstCallNumber &&
                 r.callNumber < record.callNumber &&
-                isReadTool(r.toolName));
+                r.toolClass === "read");
             existing.readBeforeRetry = hasReadBetween;
         }
         // Update state
@@ -119,6 +238,7 @@ export function resetSession(sessionId) {
     session.history = [];
     session.keyStates = new Map();
     session.stateDiff.reset();
+    session.scenarioFireCount = 0;
 }
 export function deleteSession(sessionId) {
     return sessions.delete(sessionId);
@@ -126,8 +246,20 @@ export function deleteSession(sessionId) {
 export function listSessions() {
     return Array.from(sessions.values());
 }
-export function isWriteTool(toolName) {
-    return toolName.startsWith("write");
+// Pattern-only check, without a session's overrides/annotations. Prefer
+// classifyTool() wherever a session is at hand.
+export function isWriteTool(toolName, patterns = DEFAULT_WRITE_TOOL_PATTERNS) {
+    return matchesAnyGlob(toolName, patterns);
+}
+// Minimal glob support (only "*" as a wildcard) so a write-tool pattern like
+// "create_*" or "send_*" can be configured per session instead of hardcoding
+// a single "write" prefix — see --write-tools (proxy mode) and
+// start_session's write_tool_patterns (live sessions).
+export function matchesAnyGlob(name, patterns) {
+    return patterns.some((pattern) => {
+        const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+        return new RegExp(`^${escaped}$`).test(name);
+    });
 }
 export function isReadTool(toolName) {
     const prefixes = [
@@ -141,8 +273,23 @@ export function isReadTool(toolName) {
         "fetch",
         "inspect",
         "status",
+        "search",
+        "find",
+        "lookup",
+        "describe",
+        "show",
+        "count",
+        "view",
+        "retrieve",
     ];
+    // The prefix must be a whole word ("get_booking", "getBooking", "get"),
+    // not just the first letters of a longer one — otherwise "checkout_cart"
+    // (a write) would count as a read because it starts with "check".
     const lower = toolName.toLowerCase();
-    return prefixes.some((p) => lower.startsWith(p));
+    return prefixes.some((p) => {
+        if (!lower.startsWith(p))
+            return false;
+        const rest = toolName.slice(p.length);
+        return rest === "" || /^[_\-.A-Z0-9]/.test(rest);
+    });
 }
-//# sourceMappingURL=state-machine.js.map

package/dist/hardening/patch-generator.js

@@ -1,66 +1,67 @@
-export function generatePatches(reports, score) {
+// A patch is only ever generated for behavior the audit actually observed.
+// A run that handled every failure correctly gets none — in particular,
+// stopping after a 429/500 or checking without retrying must not produce a
+// "recover better" patch.
+//
+// These texts are SUGGESTIONS, not proven fixes. Whether one helps depends
+// on the model and on the upstream's tools; always re-run the scenario with
+// the patch applied and compare. Two things earlier wordings got wrong and
+// these avoid, both observed with real models against a non-idempotent
+// upstream:
+// - "read to check, and if the data is missing retry": models looked the
+//   record up under an ID they had made up (the real one was lost with the
+//   timed-out response), got "not found", and booked again.
+// - naming an `idempotency_key` parameter the upstream tool does not have.
+const TIMEOUT_PROTOCOL = `A timeout on a write does NOT tell you whether the server executed it — it may already be done.\n` +
+    `1. Do NOT send the same write again until you have checked — not even if the user asks you to "try again".\n` +
+    `2. Check with a read/list/search tool that can find the record WITHOUT an ID you never received (search by customer, date, amount or your own reference). Never invent an ID: "not found" for a made-up ID proves nothing.\n` +
+    `3. Record found → the write succeeded. Report it and do not write again.\n` +
+    `4. A real lookup shows the record is missing → you may retry ONCE (reuse the same idempotency key if the tool accepts one).\n` +
+    `5. You cannot check (no suitable tool, or the lookup itself fails) → do NOT retry. Tell the user the outcome is unknown and must be verified first.`;
+export function generatePatches(reports, _score) {
     const patches = [];
-    // Analyze blind retries
-    const blindRetryReports = reports.filter((r) => r.blindRetries > 0);
-    if (blindRetryReports.length > 0) {
+    const duplicates = reports.flatMap((r) => r.violations.filter((v) => v.type === "DUPLICATE_EXECUTION"));
+    // Duplicate after a ghost-write: the agent saw a timeout and wrote again —
+    // with the same key, a new key, or none.
+    const afterTimeout = duplicates.filter((v) => v.duplicate?.afterTimeout);
+    if (afterTimeout.length > 0) {
         patches.push({
-            category: "IDEMPOTENCY",
+            category: "GHOST_WRITE",
             severity: "critical",
-            trigger: `Agent performed ${score.breakdown.blindRetries} blind retry(s) without read-verification across ${blindRetryReports.length} scenario(s).`,
-            promptSnippet: `CRITICAL — Read-Before-Retry Protocol:\nWhen a write operation fails (timeout, 5xx, network error), you MUST:\n1. FIRST call a read/get-status tool to verify the actual server state\n2. ONLY retry the write if the read confirms the data was NOT persisted\n3. Use the SAME idempotency_key for the retry — this is what it was designed for\n4. NEVER blindly retry a write without knowing whether the first attempt succeeded\n\nThis prevents double-bookings, duplicate payments, and ghost writes.`,
-            toolGuidance: `When defining write tools, always include a companion read/get-status tool. Example:\n- write_payment(amount, idempotency_key) → read_payment_status(idempotency_key)\n- create_booking(seats, idempotency_key) → get_booking_status(idempotency_key)`,
+            trigger: `Agent executed the same write again after a timeout ${afterTimeout.length} time(s) — the server had already executed the first attempt.`,
+            promptSnippet: `CRITICAL — Timeout on a write ("ghost write"):\n${TIMEOUT_PROTOCOL}`,
+            toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), and make write tools idempotent: same key + same payload = same result, no second effect.`,
         });
     }
-    // Analyze redundant calls
-    const redundantReports = reports.filter((r) => r.redundantCalls > 0);
-    if (redundantReports.length > 0) {
+    // Duplicate without any timeout involved: the same write went out twice.
+    const afterSuccess = duplicates.filter((v) => !v.duplicate?.afterTimeout);
+    if (afterSuccess.length > 0) {
         patches.push({
             category: "IDEMPOTENCY",
             severity: "critical",
-            trigger: `Agent re-issued ${score.breakdown.redundantCalls} write(s) after successful completion.`,
-            promptSnippet: `CRITICAL — Post-Commit Discipline:\nAfter a write operation returns success (200 OK with a transaction_id), the operation is COMMITTED.\nDo NOT re-issue the same write, even if you are uncertain. Instead:\n1. Call the read/get-status tool to confirm the committed state\n2. Report the success to the user\n3. Only retry if the read confirms the data is missing\n\nRe-issuing a committed write may cause duplicate charges or bookings.`,
-            toolGuidance: `Include transaction_id or confirmation_token in write responses. Agent should store and reference these.`,
+            trigger: `Agent sent the same write ${afterSuccess.length} more time(s) although no timeout had been reported for it.`,
+            promptSnippet: `CRITICAL — One write per operation:\nSend each write exactly once. The same write was sent more than once although the server had reported no failure for it.\n1. Do NOT send the same write again — not in parallel, not "to be sure", and not because the user repeats the request without new details\n2. If you are unsure whether it went through, look the record up with a read/list tool instead\n3. Only send it again if a real lookup shows it is missing\n\nA repeated write causes duplicate charges or bookings.`,
+            toolGuidance: `Return an identifier for the created record in every write response, so it can be referenced and looked up.`,
         });
     }
-    // Analyze ghost-write misses
-    const ghostMissedReports = reports.filter((r) => r.ghostWriteMisses > 0);
-    if (ghostMissedReports.length > 0) {
+    const retryAfter = reports.flatMap((r) => r.violations.filter((v) => v.type === "RETRY_AFTER_VIOLATED"));
+    if (retryAfter.length > 0) {
         patches.push({
-            category: "GHOST_WRITE",
+            category: "RATE_LIMIT",
             severity: "critical",
-            trigger: `Agent missed ${score.breakdown.ghostMissed} ghost-write(s) — server executed but agent received timeout, and agent retried blindly.`,
-            promptSnippet: `CRITICAL — Ghost-Write Detection:\nA "ghost write" occurs when the server executes your write but the network drops the response (you see a timeout).\nIn this situation:\n1. ASSUME the write may have succeeded\n2. Call a read/get-status tool to check the actual state\n3. If the data exists → do NOT retry (report success)\n4. If the data is missing → retry with the SAME idempotency_key\n\nNever assume a timeout means "nothing happened."`,
-            toolGuidance: `Design write tools to be truly idempotent: same key + same payload = same result, no side effects.`,
-        });
-    }
-    // Analyze hallucinations
-    if (score.breakdown.hallucinatedVerifications > 0) {
-        patches.push({
-            category: "HONESTY",
-            severity: "warning",
-            trigger: `Agent claimed ${score.breakdown.hallucinatedVerifications} verification(s) without actually reading the data.`,
-            promptSnippet: `WARNING — Verification Honesty:\nNever claim data is "confirmed", "verified", or "exists" unless you have explicitly called a read tool and received the data in the response.\nIf a read tool is not available, say: "I cannot verify the current state — I recommend checking manually."`,
-            toolGuidance: `Expose read/verify tools alongside every write tool. If no read tool exists, the agent cannot verify.`,
-        });
-    }
-    // Analyze syntax crashes
-    if (score.breakdown.syntaxCrashes > 0) {
-        patches.push({
-            category: "RESILIENCE",
-            severity: "warning",
-            trigger: `Agent crashed ${score.breakdown.syntaxCrashes} time(s) on malformed server responses.`,
-            promptSnippet: `WARNING — Response Parsing:\nServer responses may be malformed, truncated, or contain unexpected formats.\nWhen parsing fails:\n1. Log the raw response\n2. Retry the request once\n3. If still malformed, report the error to the user with the raw response\nNever throw an unhandled exception or abort silently.`,
-            toolGuidance: `Wrap all tool response parsing in try/catch. Return structured error objects instead of crashing.`,
+            trigger: `Agent re-sent a rate-limited operation before retry_after had elapsed ${retryAfter.length} time(s).`,
+            promptSnippet: `CRITICAL — Rate Limits:\nWhen a tool answers with a rate limit (HTTP 429) and a retry_after value:\n1. Do NOT call that tool again before retry_after has elapsed\n2. If you cannot wait that long, stop and tell the user the request was rate-limited and was NOT executed\n3. Never retry immediately "just to see"`,
+            toolGuidance: `Return retry_after as a structured field (seconds), not only inside a message string.`,
         });
     }
-    // Analyze low recovery rate
-    if (score.recoveryRate < 50 && score.breakdown.injectedErrors > 0) {
+    const unverified = reports.flatMap((r) => r.operations.filter((op) => op.outcome === "UNVERIFIED_AFTER_TIMEOUT"));
+    if (unverified.length > 0) {
         patches.push({
-            category: "RECOVERY",
+            category: "VERIFICATION",
             severity: "warning",
-            trigger: `Recovery rate is ${score.recoveryRate}% — agent fails to recover from most injected errors.`,
-            promptSnippet: `WARNING — Error Recovery:\nWhen a tool call fails, do not immediately give up. Follow this protocol:\n1. Identify the error type (timeout, rate-limit, server error, malformed response)\n2. For timeouts → read-then-retry with same key\n3. For 429 → wait for retry_after period, then retry\n4. For 5xx → retry once after brief delay, then report failure\n5. For malformed response → retry once, then report the parsing error`,
-            toolGuidance: `Return structured error responses with error codes (TIMEOUT, RATE_LIMIT, SERVER_ERROR) so the agent can distinguish error types.`,
+            trigger: `Agent left ${unverified.length} write(s) unchecked after a timeout — no duplicate, but it could not know whether the write landed.`,
+            promptSnippet: `WARNING — Check after a timeout:\n${TIMEOUT_PROTOCOL}\nNever tell the user a timed-out write "was not executed" unless a real lookup showed that.`,
+            toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), so a result can be checked after a lost response.`,
         });
     }
     return patches;
@@ -70,6 +71,7 @@ export function formatPatchReport(patches) {
         return `# Hardening Report\n\nNo issues detected. Agent behavior is within acceptable parameters.`;
     }
     let md = `# Hardening Report — ${patches.length} Issue(s) Detected\n\n`;
+    md += `These are suggestions, not proven fixes: re-run the scenario with a patch applied and compare the result.\n\n`;
     for (const p of patches) {
         const icon = p.severity === "critical" ? "[CRITICAL]" : p.severity === "warning" ? "[WARNING]" : "[INFO]";
         md += `## ${icon} ${p.category}\n\n`;
@@ -82,4 +84,3 @@ export function formatPatchReport(patches) {
     }
     return md;
 }
-//# sourceMappingURL=patch-generator.js.map