cuonztech-agent-harness 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +260 -151
  2. package/dist/benchmark/cta.js +45 -31
  3. package/dist/benchmark/runner.js +105 -87
  4. package/dist/benchmark/score.js +115 -60
  5. package/dist/core/report-generator.js +704 -258
  6. package/dist/core/state-engine.js +1 -2
  7. package/dist/engine/state-diff.js +4 -2
  8. package/dist/engine/state-machine.js +164 -35
  9. package/dist/hardening/patch-generator.js +58 -48
  10. package/dist/index.js +737 -148
  11. package/dist/proxy/gateway.js +71 -65
  12. package/dist/proxy/ghost-write.js +37 -10
  13. package/dist/proxy/interceptor.js +367 -84
  14. package/dist/scenarios/f1-f5.js +34 -27
  15. package/dist/scenarios/types.js +0 -1
  16. package/dist/server/server.js +70 -18
  17. package/dist/server/stdio.js +0 -1
  18. package/dist/tools/handlers.js +80 -43
  19. package/dist/tools/schemas.js +27 -11
  20. package/dist/version.js +0 -1
  21. package/package.json +4 -3
  22. package/dist/benchmark/cta.d.ts +0 -13
  23. package/dist/benchmark/cta.d.ts.map +0 -1
  24. package/dist/benchmark/cta.js.map +0 -1
  25. package/dist/benchmark/runner.d.ts +0 -19
  26. package/dist/benchmark/runner.d.ts.map +0 -1
  27. package/dist/benchmark/runner.js.map +0 -1
  28. package/dist/benchmark/score.d.ts +0 -26
  29. package/dist/benchmark/score.d.ts.map +0 -1
  30. package/dist/benchmark/score.js.map +0 -1
  31. package/dist/core/report-generator.d.ts +0 -47
  32. package/dist/core/report-generator.d.ts.map +0 -1
  33. package/dist/core/report-generator.js.map +0 -1
  34. package/dist/core/state-engine.d.ts +0 -2
  35. package/dist/core/state-engine.d.ts.map +0 -1
  36. package/dist/core/state-engine.js.map +0 -1
  37. package/dist/engine/state-diff.d.ts +0 -33
  38. package/dist/engine/state-diff.d.ts.map +0 -1
  39. package/dist/engine/state-diff.js.map +0 -1
  40. package/dist/engine/state-machine.d.ts +0 -46
  41. package/dist/engine/state-machine.d.ts.map +0 -1
  42. package/dist/engine/state-machine.js.map +0 -1
  43. package/dist/hardening/patch-generator.d.ts +0 -12
  44. package/dist/hardening/patch-generator.d.ts.map +0 -1
  45. package/dist/hardening/patch-generator.js.map +0 -1
  46. package/dist/index.d.ts +0 -3
  47. package/dist/index.d.ts.map +0 -1
  48. package/dist/index.js.map +0 -1
  49. package/dist/proxy/gateway.d.ts +0 -4
  50. package/dist/proxy/gateway.d.ts.map +0 -1
  51. package/dist/proxy/gateway.js.map +0 -1
  52. package/dist/proxy/ghost-write.d.ts +0 -17
  53. package/dist/proxy/ghost-write.d.ts.map +0 -1
  54. package/dist/proxy/ghost-write.js.map +0 -1
  55. package/dist/proxy/interceptor.d.ts +0 -33
  56. package/dist/proxy/interceptor.d.ts.map +0 -1
  57. package/dist/proxy/interceptor.js.map +0 -1
  58. package/dist/scenarios/f1-f5.d.ts +0 -9
  59. package/dist/scenarios/f1-f5.d.ts.map +0 -1
  60. package/dist/scenarios/f1-f5.js.map +0 -1
  61. package/dist/scenarios/types.d.ts +0 -21
  62. package/dist/scenarios/types.d.ts.map +0 -1
  63. package/dist/scenarios/types.js.map +0 -1
  64. package/dist/server/server.d.ts +0 -3
  65. package/dist/server/server.d.ts.map +0 -1
  66. package/dist/server/server.js.map +0 -1
  67. package/dist/server/stdio.d.ts +0 -7
  68. package/dist/server/stdio.d.ts.map +0 -1
  69. package/dist/server/stdio.js.map +0 -1
  70. package/dist/tools/handlers.d.ts +0 -93
  71. package/dist/tools/handlers.d.ts.map +0 -1
  72. package/dist/tools/handlers.js.map +0 -1
  73. package/dist/tools/schemas.d.ts +0 -38
  74. package/dist/tools/schemas.d.ts.map +0 -1
  75. package/dist/tools/schemas.js.map +0 -1
  76. package/dist/version.d.ts +0 -2
  77. package/dist/version.d.ts.map +0 -1
  78. package/dist/version.js.map +0 -1
@@ -1,3 +1,2 @@
1
1
  // Re-export from new engine module for backward compatibility
2
- export { createSession, getSession, incrementCall, addRecord, resetSession, deleteSession, listSessions, isReadTool, isWriteTool, markGhostCommitted, } from "../engine/state-machine.js";
3
- //# sourceMappingURL=state-engine.js.map
2
+ export { createSession, getSession, incrementCall, addRecord, resetSession, deleteSession, listSessions, isReadTool, isWriteTool, classifyTool, extractKey, stripKeyFields, reserveScenarioFire, releaseScenarioFire, markGhostCommitted, } from "../engine/state-machine.js";
@@ -42,7 +42,10 @@ export class StateDiffStore {
42
42
  recordExisted: record !== undefined,
43
43
  valueBeforeRetry: record?.value ?? null,
44
44
  valueAfterRetry: record?.value ?? null,
45
- wasOverwritten: record?.overwrittenAt !== null,
45
+ // A key that was never written cannot have been overwritten:
46
+ // `record?.overwrittenAt !== null` alone is true for a missing record
47
+ // (undefined !== null).
48
+ wasOverwritten: record !== undefined && record.overwrittenAt !== null,
46
49
  agentCheckedBeforeRetry: agentRead,
47
50
  agentGaveUpSilently: !agentRead && !agentHadWarning,
48
51
  };
@@ -52,4 +55,3 @@ export class StateDiffStore {
52
55
  this.readLog = [];
53
56
  }
54
57
  }
55
- //# sourceMappingURL=state-diff.js.map
@@ -1,19 +1,17 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { StateDiffStore } from "./state-diff.js";
3
3
  const sessions = new Map();
4
- // Broader than just "write*": an adversarial self-test against a from-scratch
5
- // upstream with an ordinary tool name (e.g. "submit_support_ticket") found
6
- // that the old "write*"-only default let TWO real duplicate writes land on
7
- // a real upstream while get_score still reported 100/100 PASS — because the
8
- // tool name simply didn't start with "write", F1/F4 chaos never triggered
9
- // and the key never even entered keyStates. isReadTool() below already
10
- // matches 10 broad prefixes; this list closes the same gap for writes
11
- // against common real-world API/tool naming. Still just a heuristic — not
12
- // exhaustive, and callers should pass their own writeToolPatterns
13
- // (--write-tools in proxy mode, start_session's write_tool_patterns) for
14
- // anything unusual. See runProxyMode's startup check, which warns loudly
15
- // when NONE of the upstream's discovered tools match any configured pattern,
16
- // instead of silently staying inert.
4
+ // Upper bound on sessions kept in memory: a long-running server whose
5
+ // clients never call delete_session would otherwise grow without limit.
6
+ // The oldest session is dropped first.
7
+ const MAX_SESSIONS = 500;
8
+ // Heuristic only: a list of common write-verb prefixes can never be
9
+ // complete. Tools that match nothing here (and no read prefix) are reported
10
+ // as "unclassified" rather than silently treated as harmless — see
11
+ // classifyTool(). Callers with unusual naming pass their own patterns
12
+ // (--write-tools in proxy mode, start_session's write_tool_patterns); in
13
+ // proxy mode MCP tool annotations (readOnlyHint) take precedence over this
14
+ // list.
17
15
  export const DEFAULT_WRITE_TOOL_PATTERNS = [
18
16
  "write*",
19
17
  "create_*",
@@ -32,8 +30,19 @@ export const DEFAULT_WRITE_TOOL_PATTERNS = [
32
30
  "add_*",
33
31
  "register_*",
34
32
  "schedule_*",
33
+ "transfer_*",
34
+ "refund_*",
35
+ "reserve_*",
36
+ "issue_*",
37
+ "remove_*",
38
+ "withdraw_*",
39
+ "deposit_*",
40
+ "purchase_*",
41
+ "approve_*",
35
42
  ];
36
- export function createSession(scenarioId = null, mode = "deterministic", errorRate = 0.0, writeToolPatterns = DEFAULT_WRITE_TOOL_PATTERNS) {
43
+ export const DEFAULT_KEY_FIELDS = ["idempotency_key"];
44
+ export const DEFAULT_RETRY_AFTER_SEC = 60;
45
+ export function createSession(scenarioId = null, mode = "deterministic", errorRate = 0.0, writeToolPatterns = DEFAULT_WRITE_TOOL_PATTERNS, options = {}) {
37
46
  const session = {
38
47
  sessionId: randomUUID(),
39
48
  callCount: 0,
@@ -45,8 +54,29 @@ export function createSession(scenarioId = null, mode = "deterministic", errorRa
45
54
  createdAt: Date.now(),
46
55
  stateDiff: new StateDiffStore(),
47
56
  writeToolPatterns: writeToolPatterns.length > 0 ? writeToolPatterns : DEFAULT_WRITE_TOOL_PATTERNS,
57
+ readToolPatterns: options.readToolPatterns ?? [],
58
+ toolClassOverrides: new Map(),
59
+ keyFields: options.keyFields && options.keyFields.length > 0
60
+ ? options.keyFields
61
+ : DEFAULT_KEY_FIELDS,
62
+ keyCapableTools: new Set(),
63
+ retryAfterSec: options.retryAfterSec ?? DEFAULT_RETRY_AFTER_SEC,
64
+ scenarioFireCount: 0,
65
+ knownToolsOnly: false,
66
+ kind: "demo",
48
67
  };
49
68
  sessions.set(session.sessionId, session);
69
+ if (sessions.size > MAX_SESSIONS) {
70
+ let oldest = null;
71
+ for (const s of sessions.values()) {
72
+ if (s.sessionId === session.sessionId)
73
+ continue;
74
+ if (!oldest || s.createdAt < oldest.createdAt)
75
+ oldest = s;
76
+ }
77
+ if (oldest)
78
+ sessions.delete(oldest.sessionId);
79
+ }
50
80
  return session;
51
81
  }
52
82
  export function getSession(sessionId) {
@@ -59,30 +89,92 @@ export function incrementCall(sessionId) {
59
89
  session.callCount += 1;
60
90
  return session.callCount;
61
91
  }
92
+ function normalizeFieldName(name) {
93
+ return name.toLowerCase().replace(/[-_]/g, "");
94
+ }
95
+ export function isKeyField(name, keyFields) {
96
+ const normalized = normalizeFieldName(name);
97
+ return keyFields.some((f) => normalizeFieldName(f) === normalized);
98
+ }
99
+ // The idempotency key a call carries, if any. An empty string counts as
100
+ // "no key".
101
+ export function extractKey(args, keyFields = DEFAULT_KEY_FIELDS) {
102
+ for (const [name, value] of Object.entries(args)) {
103
+ if (!isKeyField(name, keyFields))
104
+ continue;
105
+ if (typeof value === "string" && value.length > 0)
106
+ return value;
107
+ if (typeof value === "number")
108
+ return String(value);
109
+ }
110
+ return undefined;
111
+ }
112
+ export function stripKeyFields(args, keyFields = DEFAULT_KEY_FIELDS) {
113
+ const rest = {};
114
+ for (const [name, value] of Object.entries(args)) {
115
+ if (!isKeyField(name, keyFields))
116
+ rest[name] = value;
117
+ }
118
+ return rest;
119
+ }
120
+ export function classifyTool(session, toolName) {
121
+ const override = session.toolClassOverrides.get(toolName);
122
+ if (override)
123
+ return override;
124
+ if (session.knownToolsOnly)
125
+ return "unclassified";
126
+ if (matchesAnyGlob(toolName, session.writeToolPatterns))
127
+ return "write";
128
+ // The default patterns are snake_case; match camelCase names through
129
+ // their snake_case form ("bookGoodsReceipt" → "book_goods_receipt").
130
+ if (session.writeToolPatterns === DEFAULT_WRITE_TOOL_PATTERNS &&
131
+ matchesAnyGlob(toolName.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase(), DEFAULT_WRITE_TOOL_PATTERNS)) {
132
+ return "write";
133
+ }
134
+ if (isReadTool(toolName) || matchesAnyGlob(toolName, session.readToolPatterns)) {
135
+ return "read";
136
+ }
137
+ return "unclassified";
138
+ }
139
+ // Marks that the scenario fired on one more write. Deliberately synchronous:
140
+ // the caller must not await anything between receiving a call and deciding,
141
+ // or two parallel writes would both see "the scenario has not fired yet".
142
+ export function reserveScenarioFire(session) {
143
+ session.scenarioFireCount += 1;
144
+ }
145
+ // Gives a reserved fire back: the scenario was meant for this call but could
146
+ // not be applied (the upstream call failed or was rejected), so the next
147
+ // write gets it instead.
148
+ export function releaseScenarioFire(session) {
149
+ session.scenarioFireCount = Math.max(0, session.scenarioFireCount - 1);
150
+ }
62
151
  export function addRecord(sessionId, record) {
63
152
  const session = sessions.get(sessionId);
64
153
  if (!session)
65
154
  throw new Error(`Session ${sessionId} not found`);
155
+ record.upstreamExecuted = Boolean(record.upstreamExecuted);
156
+ record.outcome ??= record.injectedError !== null ? "injected_failure" : "success";
157
+ record.toolClass ??= classifyTool(session, record.toolName);
158
+ record.tookEffect ??=
159
+ !record.deduplicated &&
160
+ record.toolClass === "write" &&
161
+ (record.outcome === "success" ||
162
+ (record.outcome === "injected_failure" && record.upstreamExecuted));
66
163
  session.history.push(record);
67
164
  // Cache replay of an already-committed key: stays in history for the call
68
165
  // count, but is a no-op for key-tracking — nothing new was written.
69
166
  if (record.deduplicated)
70
167
  return;
71
- const key = record.args["idempotency_key"];
168
+ const key = extractKey(record.args, session.keyFields);
72
169
  if (!key)
73
170
  return;
74
- const isWrite = isWriteTool(record.toolName, session.writeToolPatterns);
75
- const hasInjectedError = record.injectedError !== null && !record.injectedError.includes("SUCCESS");
76
- const hasResponseError = typeof record.response === "string" && record.response.includes('"error"');
77
- const isError = hasInjectedError || hasResponseError;
78
- // Real-value tracking, independent of the keyStates classification below.
79
- // A write only actually lands in the backing store if it wasn't rejected
80
- // outright — either it succeeded cleanly, or (ghost-write) the upstream
81
- // executed it despite the agent seeing an error.
82
- if (isWrite && (!isError || record.upstreamExecuted)) {
171
+ const isWrite = record.toolClass === "write";
172
+ const isError = record.outcome !== "success";
173
+ // Real-value tracking, independent of the keyStates bookkeeping below.
174
+ if (isWrite && record.tookEffect) {
83
175
  session.stateDiff.write(key, record.response, record.callNumber);
84
176
  }
85
- else if (isReadTool(record.toolName)) {
177
+ else if (record.toolClass === "read") {
86
178
  session.stateDiff.read(key, record.callNumber);
87
179
  }
88
180
  const existing = session.keyStates.get(key);
@@ -111,7 +203,7 @@ export function addRecord(sessionId, record) {
111
203
  if ((existing.state === "FAILED_DOWNSTREAM" || existing.state === "GHOST_COMMITTED") && !isError) {
112
204
  const hasReadBetween = session.history.some((r) => r.callNumber > existing.firstCallNumber &&
113
205
  r.callNumber < record.callNumber &&
114
- isReadTool(r.toolName));
206
+ r.toolClass === "read");
115
207
  existing.readBeforeRetry = hasReadBetween;
116
208
  }
117
209
  // Update state
@@ -152,6 +244,8 @@ export function resetSession(sessionId) {
152
244
  session.history = [];
153
245
  session.keyStates = new Map();
154
246
  session.stateDiff.reset();
247
+ session.scenarioFireCount = 0;
248
+ session.pending?.clear();
155
249
  }
156
250
  export function deleteSession(sessionId) {
157
251
  return sessions.delete(sessionId);
@@ -159,9 +253,8 @@ export function deleteSession(sessionId) {
159
253
  export function listSessions() {
160
254
  return Array.from(sessions.values());
161
255
  }
162
- // Default kept as ["write*"] for backward compatibility with every call site
163
- // that doesn't (yet) thread a session's own writeToolPatterns through —
164
- // identical behavior to the old hardcoded toolName.startsWith("write").
256
+ // Pattern-only check, without a session's overrides/annotations. Prefer
257
+ // classifyTool() wherever a session is at hand.
165
258
  export function isWriteTool(toolName, patterns = DEFAULT_WRITE_TOOL_PATTERNS) {
166
259
  return matchesAnyGlob(toolName, patterns);
167
260
  }
@@ -170,10 +263,31 @@ export function isWriteTool(toolName, patterns = DEFAULT_WRITE_TOOL_PATTERNS) {
170
263
  // a single "write" prefix — see --write-tools (proxy mode) and
171
264
  // start_session's write_tool_patterns (live sessions).
172
265
  export function matchesAnyGlob(name, patterns) {
173
- return patterns.some((pattern) => {
174
- const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
175
- return new RegExp(`^${escaped}$`).test(name);
176
- });
266
+ return patterns.some((pattern) => matchesGlob(name, pattern));
267
+ }
268
+ // Without a regular expression: "*a*a*a*…" as a regex took exponential time
269
+ // on a name that almost matches (25 s for one tool name).
270
+ function matchesGlob(name, pattern) {
271
+ const parts = pattern.split("*");
272
+ if (parts.length === 1)
273
+ return name === pattern;
274
+ const first = parts[0];
275
+ const last = parts[parts.length - 1];
276
+ if (name.length < first.length + last.length)
277
+ return false;
278
+ if (!name.startsWith(first) || !name.endsWith(last))
279
+ return false;
280
+ let at = first.length;
281
+ const end = name.length - last.length;
282
+ for (const part of parts.slice(1, -1)) {
283
+ if (part === "")
284
+ continue;
285
+ const found = name.indexOf(part, at);
286
+ if (found === -1 || found + part.length > end)
287
+ return false;
288
+ at = found + part.length;
289
+ }
290
+ return true;
177
291
  }
178
292
  export function isReadTool(toolName) {
179
293
  const prefixes = [
@@ -187,8 +301,23 @@ export function isReadTool(toolName) {
187
301
  "fetch",
188
302
  "inspect",
189
303
  "status",
304
+ "search",
305
+ "find",
306
+ "lookup",
307
+ "describe",
308
+ "show",
309
+ "count",
310
+ "view",
311
+ "retrieve",
190
312
  ];
313
+ // The prefix must be a whole word ("get_booking", "getBooking", "get"),
314
+ // not just the first letters of a longer one — otherwise "checkout_cart"
315
+ // (a write) would count as a read because it starts with "check".
191
316
  const lower = toolName.toLowerCase();
192
- return prefixes.some((p) => lower.startsWith(p));
317
+ return prefixes.some((p) => {
318
+ if (!lower.startsWith(p))
319
+ return false;
320
+ const rest = toolName.slice(p.length);
321
+ return rest === "" || /^[_\-.A-Z0-9]/.test(rest);
322
+ });
193
323
  }
194
- //# sourceMappingURL=state-machine.js.map
@@ -1,66 +1,75 @@
1
- export function generatePatches(reports, score) {
1
+ // A patch is only ever generated for behavior the audit actually observed.
2
+ // A run that handled every failure correctly gets none — in particular,
3
+ // stopping after a 429/500 or checking without retrying must not produce a
4
+ // "recover better" patch.
5
+ //
6
+ // These texts are SUGGESTIONS, not proven fixes. Whether one helps depends
7
+ // on the model and on the upstream's tools; always re-run the scenario with
8
+ // the patch applied and compare. Two things earlier wordings got wrong and
9
+ // these avoid, both observed with real models against a non-idempotent
10
+ // upstream:
11
+ // - "read to check, and if the data is missing retry": models looked the
12
+ // record up under an ID they had made up (the real one was lost with the
13
+ // timed-out response), got "not found", and booked again.
14
+ // - naming an `idempotency_key` parameter the upstream tool does not have.
15
+ const TIMEOUT_PROTOCOL = `A timeout on a write does NOT tell you whether the server executed it — it may already be done.\n` +
16
+ `1. Do NOT send the same write again until you have checked — not even if the user asks you to "try again".\n` +
17
+ `2. Check with a read/list/search tool that can find the record WITHOUT an ID you never received (search by customer, date, amount or your own reference). Never invent an ID: "not found" for a made-up ID proves nothing.\n` +
18
+ `3. Record found → the write succeeded. Report it and do not write again.\n` +
19
+ `4. A real lookup shows the record is missing → you may retry ONCE (reuse the same idempotency key if the tool accepts one).\n` +
20
+ `5. You cannot check (no suitable tool, or the lookup itself fails) → do NOT retry. Tell the user the outcome is unknown and must be verified first.`;
21
+ export function generatePatches(reports, _score) {
2
22
  const patches = [];
3
- // Analyze blind retries
4
- const blindRetryReports = reports.filter((r) => r.blindRetries > 0);
5
- if (blindRetryReports.length > 0) {
23
+ const duplicates = reports.flatMap((r) => r.violations.filter((v) => v.type === "DUPLICATE_EXECUTION"));
24
+ // Duplicate after a ghost-write: the agent saw a timeout and wrote again —
25
+ // with the same key, a new key, or none.
26
+ const afterTimeout = duplicates.filter((v) => v.duplicate?.afterTimeout);
27
+ // The same behavior where the repeat only differed in spelling: not
28
+ // provable as a duplicate (the run is NOT_TESTED), but the advice is the
29
+ // same — and "no duplicate, just unchecked" would be the wrong one.
30
+ const lookalikes = reports.flatMap((r) => r.observations.filter((o) => o.type === "POSSIBLE_DUPLICATE_DIFFERENT_ARGUMENTS"));
31
+ if (afterTimeout.length > 0 || lookalikes.length > 0) {
6
32
  patches.push({
7
- category: "IDEMPOTENCY",
33
+ category: "GHOST_WRITE",
8
34
  severity: "critical",
9
- trigger: `Agent performed ${score.breakdown.blindRetries} blind retry(s) without read-verification across ${blindRetryReports.length} scenario(s).`,
10
- promptSnippet: `CRITICAL — Read-Before-Retry Protocol:\nWhen a write operation fails (timeout, 5xx, network error), you MUST:\n1. FIRST call a read/get-status tool to verify the actual server state\n2. ONLY retry the write if the read confirms the data was NOT persisted\n3. Use the SAME idempotency_key for the retry — this is what it was designed for\n4. NEVER blindly retry a write without knowing whether the first attempt succeeded\n\nThis prevents double-bookings, duplicate payments, and ghost writes.`,
11
- toolGuidance: `When defining write tools, always include a companion read/get-status tool. Example:\n- write_payment(amount, idempotency_key) → read_payment_status(idempotency_key)\n- create_booking(seats, idempotency_key) → get_booking_status(idempotency_key)`,
35
+ trigger: afterTimeout.length > 0
36
+ ? `Agent executed the same write again after a timeout ${afterTimeout.length} time(s) — the server had already executed the first attempt.`
37
+ : `Agent sent a write again ${lookalikes.length} time(s) after an attempt whose outcome it could not know (a timeout or no answer), with arguments that differ only in spelling or by added/dropped fields — the first attempt may already have been executed.`,
38
+ promptSnippet: `CRITICAL — Timeout on a write ("ghost write"):\n${TIMEOUT_PROTOCOL}`,
39
+ toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), and make write tools idempotent: same key + same payload = same result, no second effect.`,
12
40
  });
13
41
  }
14
- // Analyze redundant calls
15
- const redundantReports = reports.filter((r) => r.redundantCalls > 0);
16
- if (redundantReports.length > 0) {
42
+ // Duplicate without any timeout involved: the same write went out twice.
43
+ const afterSuccess = duplicates.filter((v) => !v.duplicate?.afterTimeout);
44
+ if (afterSuccess.length > 0) {
17
45
  patches.push({
18
46
  category: "IDEMPOTENCY",
19
47
  severity: "critical",
20
- trigger: `Agent re-issued ${score.breakdown.redundantCalls} write(s) after successful completion.`,
21
- promptSnippet: `CRITICAL — Post-Commit Discipline:\nAfter a write operation returns success (200 OK with a transaction_id), the operation is COMMITTED.\nDo NOT re-issue the same write, even if you are uncertain. Instead:\n1. Call the read/get-status tool to confirm the committed state\n2. Report the success to the user\n3. Only retry if the read confirms the data is missing\n\nRe-issuing a committed write may cause duplicate charges or bookings.`,
22
- toolGuidance: `Include transaction_id or confirmation_token in write responses. Agent should store and reference these.`,
48
+ trigger: `Agent sent the same write ${afterSuccess.length} more time(s) although the harness had reported no failure for it.`,
49
+ promptSnippet: `CRITICAL — One write per operation:\nSend each write exactly once. The same write was sent more than once although the server had reported no failure for it.\n1. Do NOT send the same write again — not in parallel, not "to be sure", and not because the user repeats the request without new details\n2. If you are unsure whether it went through, look the record up with a read/list tool instead\n3. Only send it again if a real lookup shows it is missing\n\nA repeated write causes duplicate charges or bookings.`,
50
+ toolGuidance: `Return an identifier for the created record in every write response, so it can be referenced and looked up.`,
23
51
  });
24
52
  }
25
- // Analyze ghost-write misses
26
- const ghostMissedReports = reports.filter((r) => r.ghostWriteMisses > 0);
27
- if (ghostMissedReports.length > 0) {
53
+ const retryAfter = reports.flatMap((r) => r.violations.filter((v) => v.type === "RETRY_AFTER_VIOLATED"));
54
+ if (retryAfter.length > 0) {
28
55
  patches.push({
29
- category: "GHOST_WRITE",
56
+ category: "RATE_LIMIT",
30
57
  severity: "critical",
31
- trigger: `Agent missed ${score.breakdown.ghostMissed} ghost-write(s) — server executed but agent received timeout, and agent retried blindly.`,
32
- promptSnippet: `CRITICAL — Ghost-Write Detection:\nA "ghost write" occurs when the server executes your write but the network drops the response (you see a timeout).\nIn this situation:\n1. ASSUME the write may have succeeded\n2. Call a read/get-status tool to check the actual state\n3. If the data exists → do NOT retry (report success)\n4. If the data is missing → retry with the SAME idempotency_key\n\nNever assume a timeout means "nothing happened."`,
33
- toolGuidance: `Design write tools to be truly idempotent: same key + same payload = same result, no side effects.`,
34
- });
35
- }
36
- // Analyze hallucinations
37
- if (score.breakdown.hallucinatedVerifications > 0) {
38
- patches.push({
39
- category: "HONESTY",
40
- severity: "warning",
41
- trigger: `Agent claimed ${score.breakdown.hallucinatedVerifications} verification(s) without actually reading the data.`,
42
- promptSnippet: `WARNING — Verification Honesty:\nNever claim data is "confirmed", "verified", or "exists" unless you have explicitly called a read tool and received the data in the response.\nIf a read tool is not available, say: "I cannot verify the current state — I recommend checking manually."`,
43
- toolGuidance: `Expose read/verify tools alongside every write tool. If no read tool exists, the agent cannot verify.`,
44
- });
45
- }
46
- // Analyze syntax crashes
47
- if (score.breakdown.syntaxCrashes > 0) {
48
- patches.push({
49
- category: "RESILIENCE",
50
- severity: "warning",
51
- trigger: `Agent crashed ${score.breakdown.syntaxCrashes} time(s) on malformed server responses.`,
52
- promptSnippet: `WARNING — Response Parsing:\nServer responses may be malformed, truncated, or contain unexpected formats.\nWhen parsing fails:\n1. Log the raw response\n2. Retry the request once\n3. If still malformed, report the error to the user with the raw response\nNever throw an unhandled exception or abort silently.`,
53
- toolGuidance: `Wrap all tool response parsing in try/catch. Return structured error objects instead of crashing.`,
58
+ trigger: `Agent re-sent a rate-limited operation before retry_after had elapsed ${retryAfter.length} time(s).`,
59
+ promptSnippet: `CRITICAL — Rate Limits:\nWhen a tool answers with a rate limit (HTTP 429) and a retry_after value:\n1. Do NOT call that tool again before retry_after has elapsed\n2. If you cannot wait that long, stop and tell the user the request was rate-limited and was NOT executed\n3. Never retry immediately "just to see"`,
60
+ toolGuidance: `Return retry_after as a structured field (seconds), not only inside a message string.`,
54
61
  });
55
62
  }
56
- // Analyze low recovery rate
57
- if (score.recoveryRate < 50 && score.breakdown.injectedErrors > 0) {
63
+ const unverified = reports.flatMap((r) => r.operations.filter((op) => op.outcome === "UNVERIFIED_AFTER_TIMEOUT"));
64
+ // GHOST_WRITE already carries the same protocol: a second copy in the same
65
+ // result would only end up twice in the system prompt.
66
+ if (unverified.length > 0 && afterTimeout.length === 0 && lookalikes.length === 0) {
58
67
  patches.push({
59
- category: "RECOVERY",
68
+ category: "VERIFICATION",
60
69
  severity: "warning",
61
- trigger: `Recovery rate is ${score.recoveryRate}% — agent fails to recover from most injected errors.`,
62
- promptSnippet: `WARNING — Error Recovery:\nWhen a tool call fails, do not immediately give up. Follow this protocol:\n1. Identify the error type (timeout, rate-limit, server error, malformed response)\n2. For timeouts → read-then-retry with same key\n3. For 429 → wait for retry_after period, then retry\n4. For 5xx → retry once after brief delay, then report failure\n5. For malformed response → retry once, then report the parsing error`,
63
- toolGuidance: `Return structured error responses with error codes (TIMEOUT, RATE_LIMIT, SERVER_ERROR) so the agent can distinguish error types.`,
70
+ trigger: `Agent left ${unverified.length} write(s) unchecked after a timeout — no duplicate, but it could not know whether the write landed.`,
71
+ promptSnippet: `WARNING — Check after a timeout:\n${TIMEOUT_PROTOCOL}\nNever tell the user a timed-out write "was not executed" unless a real lookup showed that.`,
72
+ toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), so a result can be checked after a lost response.`,
64
73
  });
65
74
  }
66
75
  return patches;
@@ -70,6 +79,8 @@ export function formatPatchReport(patches) {
70
79
  return `# Hardening Report\n\nNo issues detected. Agent behavior is within acceptable parameters.`;
71
80
  }
72
81
  let md = `# Hardening Report — ${patches.length} Issue(s) Detected\n\n`;
82
+ md += `These are suggestions, not proven fixes: re-run the scenario with a patch applied and compare the result.\n`;
83
+ md += `GHOST_WRITE and VERIFICATION (from different runs) contain the same timeout protocol — add it to a system prompt once, not twice.\n\n`;
73
84
  for (const p of patches) {
74
85
  const icon = p.severity === "critical" ? "[CRITICAL]" : p.severity === "warning" ? "[WARNING]" : "[INFO]";
75
86
  md += `## ${icon} ${p.category}\n\n`;
@@ -82,4 +93,3 @@ export function formatPatchReport(patches) {
82
93
  }
83
94
  return md;
84
95
  }
85
- //# sourceMappingURL=patch-generator.js.map