cuonztech-agent-harness 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +260 -151
- package/dist/benchmark/cta.js +45 -31
- package/dist/benchmark/runner.js +105 -87
- package/dist/benchmark/score.js +115 -60
- package/dist/core/report-generator.js +704 -258
- package/dist/core/state-engine.js +1 -2
- package/dist/engine/state-diff.js +4 -2
- package/dist/engine/state-machine.js +164 -35
- package/dist/hardening/patch-generator.js +58 -48
- package/dist/index.js +737 -148
- package/dist/proxy/gateway.js +71 -65
- package/dist/proxy/ghost-write.js +37 -10
- package/dist/proxy/interceptor.js +367 -84
- package/dist/scenarios/f1-f5.js +34 -27
- package/dist/scenarios/types.js +0 -1
- package/dist/server/server.js +70 -18
- package/dist/server/stdio.js +0 -1
- package/dist/tools/handlers.js +80 -43
- package/dist/tools/schemas.js +27 -11
- package/dist/version.js +0 -1
- package/package.json +4 -3
- package/dist/benchmark/cta.d.ts +0 -13
- package/dist/benchmark/cta.d.ts.map +0 -1
- package/dist/benchmark/cta.js.map +0 -1
- package/dist/benchmark/runner.d.ts +0 -19
- package/dist/benchmark/runner.d.ts.map +0 -1
- package/dist/benchmark/runner.js.map +0 -1
- package/dist/benchmark/score.d.ts +0 -26
- package/dist/benchmark/score.d.ts.map +0 -1
- package/dist/benchmark/score.js.map +0 -1
- package/dist/core/report-generator.d.ts +0 -47
- package/dist/core/report-generator.d.ts.map +0 -1
- package/dist/core/report-generator.js.map +0 -1
- package/dist/core/state-engine.d.ts +0 -2
- package/dist/core/state-engine.d.ts.map +0 -1
- package/dist/core/state-engine.js.map +0 -1
- package/dist/engine/state-diff.d.ts +0 -33
- package/dist/engine/state-diff.d.ts.map +0 -1
- package/dist/engine/state-diff.js.map +0 -1
- package/dist/engine/state-machine.d.ts +0 -46
- package/dist/engine/state-machine.d.ts.map +0 -1
- package/dist/engine/state-machine.js.map +0 -1
- package/dist/hardening/patch-generator.d.ts +0 -12
- package/dist/hardening/patch-generator.d.ts.map +0 -1
- package/dist/hardening/patch-generator.js.map +0 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/proxy/gateway.d.ts +0 -4
- package/dist/proxy/gateway.d.ts.map +0 -1
- package/dist/proxy/gateway.js.map +0 -1
- package/dist/proxy/ghost-write.d.ts +0 -17
- package/dist/proxy/ghost-write.d.ts.map +0 -1
- package/dist/proxy/ghost-write.js.map +0 -1
- package/dist/proxy/interceptor.d.ts +0 -33
- package/dist/proxy/interceptor.d.ts.map +0 -1
- package/dist/proxy/interceptor.js.map +0 -1
- package/dist/scenarios/f1-f5.d.ts +0 -9
- package/dist/scenarios/f1-f5.d.ts.map +0 -1
- package/dist/scenarios/f1-f5.js.map +0 -1
- package/dist/scenarios/types.d.ts +0 -21
- package/dist/scenarios/types.d.ts.map +0 -1
- package/dist/scenarios/types.js.map +0 -1
- package/dist/server/server.d.ts +0 -3
- package/dist/server/server.d.ts.map +0 -1
- package/dist/server/server.js.map +0 -1
- package/dist/server/stdio.d.ts +0 -7
- package/dist/server/stdio.d.ts.map +0 -1
- package/dist/server/stdio.js.map +0 -1
- package/dist/tools/handlers.d.ts +0 -93
- package/dist/tools/handlers.d.ts.map +0 -1
- package/dist/tools/handlers.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -38
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js.map +0 -1
- package/dist/version.d.ts +0 -2
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js.map +0 -1
|
@@ -1,3 +1,2 @@
|
|
|
1
1
|
// Re-export from new engine module for backward compatibility
|
|
2
|
-
export { createSession, getSession, incrementCall, addRecord, resetSession, deleteSession, listSessions, isReadTool, isWriteTool, markGhostCommitted, } from "../engine/state-machine.js";
|
|
3
|
-
//# sourceMappingURL=state-engine.js.map
|
|
2
|
+
export { createSession, getSession, incrementCall, addRecord, resetSession, deleteSession, listSessions, isReadTool, isWriteTool, classifyTool, extractKey, stripKeyFields, reserveScenarioFire, releaseScenarioFire, markGhostCommitted, } from "../engine/state-machine.js";
|
|
@@ -42,7 +42,10 @@ export class StateDiffStore {
|
|
|
42
42
|
recordExisted: record !== undefined,
|
|
43
43
|
valueBeforeRetry: record?.value ?? null,
|
|
44
44
|
valueAfterRetry: record?.value ?? null,
|
|
45
|
-
|
|
45
|
+
// A key that was never written cannot have been overwritten:
|
|
46
|
+
// `record?.overwrittenAt !== null` alone is true for a missing record
|
|
47
|
+
// (undefined !== null).
|
|
48
|
+
wasOverwritten: record !== undefined && record.overwrittenAt !== null,
|
|
46
49
|
agentCheckedBeforeRetry: agentRead,
|
|
47
50
|
agentGaveUpSilently: !agentRead && !agentHadWarning,
|
|
48
51
|
};
|
|
@@ -52,4 +55,3 @@ export class StateDiffStore {
|
|
|
52
55
|
this.readLog = [];
|
|
53
56
|
}
|
|
54
57
|
}
|
|
55
|
-
//# sourceMappingURL=state-diff.js.map
|
|
@@ -1,19 +1,17 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { StateDiffStore } from "./state-diff.js";
|
|
3
3
|
const sessions = new Map();
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
// when NONE of the upstream's discovered tools match any configured pattern,
|
|
16
|
-
// instead of silently staying inert.
|
|
4
|
+
// Upper bound on sessions kept in memory: a long-running server whose
|
|
5
|
+
// clients never call delete_session would otherwise grow without limit.
|
|
6
|
+
// The oldest session is dropped first.
|
|
7
|
+
const MAX_SESSIONS = 500;
|
|
8
|
+
// Heuristic only: a list of common write-verb prefixes can never be
|
|
9
|
+
// complete. Tools that match nothing here (and no read prefix) are reported
|
|
10
|
+
// as "unclassified" rather than silently treated as harmless — see
|
|
11
|
+
// classifyTool(). Callers with unusual naming pass their own patterns
|
|
12
|
+
// (--write-tools in proxy mode, start_session's write_tool_patterns); in
|
|
13
|
+
// proxy mode MCP tool annotations (readOnlyHint) take precedence over this
|
|
14
|
+
// list.
|
|
17
15
|
export const DEFAULT_WRITE_TOOL_PATTERNS = [
|
|
18
16
|
"write*",
|
|
19
17
|
"create_*",
|
|
@@ -32,8 +30,19 @@ export const DEFAULT_WRITE_TOOL_PATTERNS = [
|
|
|
32
30
|
"add_*",
|
|
33
31
|
"register_*",
|
|
34
32
|
"schedule_*",
|
|
33
|
+
"transfer_*",
|
|
34
|
+
"refund_*",
|
|
35
|
+
"reserve_*",
|
|
36
|
+
"issue_*",
|
|
37
|
+
"remove_*",
|
|
38
|
+
"withdraw_*",
|
|
39
|
+
"deposit_*",
|
|
40
|
+
"purchase_*",
|
|
41
|
+
"approve_*",
|
|
35
42
|
];
|
|
36
|
-
export
|
|
43
|
+
export const DEFAULT_KEY_FIELDS = ["idempotency_key"];
|
|
44
|
+
export const DEFAULT_RETRY_AFTER_SEC = 60;
|
|
45
|
+
export function createSession(scenarioId = null, mode = "deterministic", errorRate = 0.0, writeToolPatterns = DEFAULT_WRITE_TOOL_PATTERNS, options = {}) {
|
|
37
46
|
const session = {
|
|
38
47
|
sessionId: randomUUID(),
|
|
39
48
|
callCount: 0,
|
|
@@ -45,8 +54,29 @@ export function createSession(scenarioId = null, mode = "deterministic", errorRa
|
|
|
45
54
|
createdAt: Date.now(),
|
|
46
55
|
stateDiff: new StateDiffStore(),
|
|
47
56
|
writeToolPatterns: writeToolPatterns.length > 0 ? writeToolPatterns : DEFAULT_WRITE_TOOL_PATTERNS,
|
|
57
|
+
readToolPatterns: options.readToolPatterns ?? [],
|
|
58
|
+
toolClassOverrides: new Map(),
|
|
59
|
+
keyFields: options.keyFields && options.keyFields.length > 0
|
|
60
|
+
? options.keyFields
|
|
61
|
+
: DEFAULT_KEY_FIELDS,
|
|
62
|
+
keyCapableTools: new Set(),
|
|
63
|
+
retryAfterSec: options.retryAfterSec ?? DEFAULT_RETRY_AFTER_SEC,
|
|
64
|
+
scenarioFireCount: 0,
|
|
65
|
+
knownToolsOnly: false,
|
|
66
|
+
kind: "demo",
|
|
48
67
|
};
|
|
49
68
|
sessions.set(session.sessionId, session);
|
|
69
|
+
if (sessions.size > MAX_SESSIONS) {
|
|
70
|
+
let oldest = null;
|
|
71
|
+
for (const s of sessions.values()) {
|
|
72
|
+
if (s.sessionId === session.sessionId)
|
|
73
|
+
continue;
|
|
74
|
+
if (!oldest || s.createdAt < oldest.createdAt)
|
|
75
|
+
oldest = s;
|
|
76
|
+
}
|
|
77
|
+
if (oldest)
|
|
78
|
+
sessions.delete(oldest.sessionId);
|
|
79
|
+
}
|
|
50
80
|
return session;
|
|
51
81
|
}
|
|
52
82
|
export function getSession(sessionId) {
|
|
@@ -59,30 +89,92 @@ export function incrementCall(sessionId) {
|
|
|
59
89
|
session.callCount += 1;
|
|
60
90
|
return session.callCount;
|
|
61
91
|
}
|
|
92
|
+
function normalizeFieldName(name) {
|
|
93
|
+
return name.toLowerCase().replace(/[-_]/g, "");
|
|
94
|
+
}
|
|
95
|
+
export function isKeyField(name, keyFields) {
|
|
96
|
+
const normalized = normalizeFieldName(name);
|
|
97
|
+
return keyFields.some((f) => normalizeFieldName(f) === normalized);
|
|
98
|
+
}
|
|
99
|
+
// The idempotency key a call carries, if any. An empty string counts as
|
|
100
|
+
// "no key".
|
|
101
|
+
export function extractKey(args, keyFields = DEFAULT_KEY_FIELDS) {
|
|
102
|
+
for (const [name, value] of Object.entries(args)) {
|
|
103
|
+
if (!isKeyField(name, keyFields))
|
|
104
|
+
continue;
|
|
105
|
+
if (typeof value === "string" && value.length > 0)
|
|
106
|
+
return value;
|
|
107
|
+
if (typeof value === "number")
|
|
108
|
+
return String(value);
|
|
109
|
+
}
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
export function stripKeyFields(args, keyFields = DEFAULT_KEY_FIELDS) {
|
|
113
|
+
const rest = {};
|
|
114
|
+
for (const [name, value] of Object.entries(args)) {
|
|
115
|
+
if (!isKeyField(name, keyFields))
|
|
116
|
+
rest[name] = value;
|
|
117
|
+
}
|
|
118
|
+
return rest;
|
|
119
|
+
}
|
|
120
|
+
export function classifyTool(session, toolName) {
|
|
121
|
+
const override = session.toolClassOverrides.get(toolName);
|
|
122
|
+
if (override)
|
|
123
|
+
return override;
|
|
124
|
+
if (session.knownToolsOnly)
|
|
125
|
+
return "unclassified";
|
|
126
|
+
if (matchesAnyGlob(toolName, session.writeToolPatterns))
|
|
127
|
+
return "write";
|
|
128
|
+
// The default patterns are snake_case; match camelCase names through
|
|
129
|
+
// their snake_case form ("bookGoodsReceipt" → "book_goods_receipt").
|
|
130
|
+
if (session.writeToolPatterns === DEFAULT_WRITE_TOOL_PATTERNS &&
|
|
131
|
+
matchesAnyGlob(toolName.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase(), DEFAULT_WRITE_TOOL_PATTERNS)) {
|
|
132
|
+
return "write";
|
|
133
|
+
}
|
|
134
|
+
if (isReadTool(toolName) || matchesAnyGlob(toolName, session.readToolPatterns)) {
|
|
135
|
+
return "read";
|
|
136
|
+
}
|
|
137
|
+
return "unclassified";
|
|
138
|
+
}
|
|
139
|
+
// Marks that the scenario fired on one more write. Deliberately synchronous:
|
|
140
|
+
// the caller must not await anything between receiving a call and deciding,
|
|
141
|
+
// or two parallel writes would both see "the scenario has not fired yet".
|
|
142
|
+
export function reserveScenarioFire(session) {
|
|
143
|
+
session.scenarioFireCount += 1;
|
|
144
|
+
}
|
|
145
|
+
// Gives a reserved fire back: the scenario was meant for this call but could
|
|
146
|
+
// not be applied (the upstream call failed or was rejected), so the next
|
|
147
|
+
// write gets it instead.
|
|
148
|
+
export function releaseScenarioFire(session) {
|
|
149
|
+
session.scenarioFireCount = Math.max(0, session.scenarioFireCount - 1);
|
|
150
|
+
}
|
|
62
151
|
export function addRecord(sessionId, record) {
|
|
63
152
|
const session = sessions.get(sessionId);
|
|
64
153
|
if (!session)
|
|
65
154
|
throw new Error(`Session ${sessionId} not found`);
|
|
155
|
+
record.upstreamExecuted = Boolean(record.upstreamExecuted);
|
|
156
|
+
record.outcome ??= record.injectedError !== null ? "injected_failure" : "success";
|
|
157
|
+
record.toolClass ??= classifyTool(session, record.toolName);
|
|
158
|
+
record.tookEffect ??=
|
|
159
|
+
!record.deduplicated &&
|
|
160
|
+
record.toolClass === "write" &&
|
|
161
|
+
(record.outcome === "success" ||
|
|
162
|
+
(record.outcome === "injected_failure" && record.upstreamExecuted));
|
|
66
163
|
session.history.push(record);
|
|
67
164
|
// Cache replay of an already-committed key: stays in history for the call
|
|
68
165
|
// count, but is a no-op for key-tracking — nothing new was written.
|
|
69
166
|
if (record.deduplicated)
|
|
70
167
|
return;
|
|
71
|
-
const key = record.args
|
|
168
|
+
const key = extractKey(record.args, session.keyFields);
|
|
72
169
|
if (!key)
|
|
73
170
|
return;
|
|
74
|
-
const isWrite =
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
// Real-value tracking, independent of the keyStates classification below.
|
|
79
|
-
// A write only actually lands in the backing store if it wasn't rejected
|
|
80
|
-
// outright — either it succeeded cleanly, or (ghost-write) the upstream
|
|
81
|
-
// executed it despite the agent seeing an error.
|
|
82
|
-
if (isWrite && (!isError || record.upstreamExecuted)) {
|
|
171
|
+
const isWrite = record.toolClass === "write";
|
|
172
|
+
const isError = record.outcome !== "success";
|
|
173
|
+
// Real-value tracking, independent of the keyStates bookkeeping below.
|
|
174
|
+
if (isWrite && record.tookEffect) {
|
|
83
175
|
session.stateDiff.write(key, record.response, record.callNumber);
|
|
84
176
|
}
|
|
85
|
-
else if (
|
|
177
|
+
else if (record.toolClass === "read") {
|
|
86
178
|
session.stateDiff.read(key, record.callNumber);
|
|
87
179
|
}
|
|
88
180
|
const existing = session.keyStates.get(key);
|
|
@@ -111,7 +203,7 @@ export function addRecord(sessionId, record) {
|
|
|
111
203
|
if ((existing.state === "FAILED_DOWNSTREAM" || existing.state === "GHOST_COMMITTED") && !isError) {
|
|
112
204
|
const hasReadBetween = session.history.some((r) => r.callNumber > existing.firstCallNumber &&
|
|
113
205
|
r.callNumber < record.callNumber &&
|
|
114
|
-
|
|
206
|
+
r.toolClass === "read");
|
|
115
207
|
existing.readBeforeRetry = hasReadBetween;
|
|
116
208
|
}
|
|
117
209
|
// Update state
|
|
@@ -152,6 +244,8 @@ export function resetSession(sessionId) {
|
|
|
152
244
|
session.history = [];
|
|
153
245
|
session.keyStates = new Map();
|
|
154
246
|
session.stateDiff.reset();
|
|
247
|
+
session.scenarioFireCount = 0;
|
|
248
|
+
session.pending?.clear();
|
|
155
249
|
}
|
|
156
250
|
export function deleteSession(sessionId) {
|
|
157
251
|
return sessions.delete(sessionId);
|
|
@@ -159,9 +253,8 @@ export function deleteSession(sessionId) {
|
|
|
159
253
|
export function listSessions() {
|
|
160
254
|
return Array.from(sessions.values());
|
|
161
255
|
}
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
// identical behavior to the old hardcoded toolName.startsWith("write").
|
|
256
|
+
// Pattern-only check, without a session's overrides/annotations. Prefer
|
|
257
|
+
// classifyTool() wherever a session is at hand.
|
|
165
258
|
export function isWriteTool(toolName, patterns = DEFAULT_WRITE_TOOL_PATTERNS) {
|
|
166
259
|
return matchesAnyGlob(toolName, patterns);
|
|
167
260
|
}
|
|
@@ -170,10 +263,31 @@ export function isWriteTool(toolName, patterns = DEFAULT_WRITE_TOOL_PATTERNS) {
|
|
|
170
263
|
// a single "write" prefix — see --write-tools (proxy mode) and
|
|
171
264
|
// start_session's write_tool_patterns (live sessions).
|
|
172
265
|
export function matchesAnyGlob(name, patterns) {
|
|
173
|
-
return patterns.some((pattern) =>
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
266
|
+
return patterns.some((pattern) => matchesGlob(name, pattern));
|
|
267
|
+
}
|
|
268
|
+
// Without a regular expression: "*a*a*a*…" as a regex took exponential time
|
|
269
|
+
// on a name that almost matches (25 s for one tool name).
|
|
270
|
+
function matchesGlob(name, pattern) {
|
|
271
|
+
const parts = pattern.split("*");
|
|
272
|
+
if (parts.length === 1)
|
|
273
|
+
return name === pattern;
|
|
274
|
+
const first = parts[0];
|
|
275
|
+
const last = parts[parts.length - 1];
|
|
276
|
+
if (name.length < first.length + last.length)
|
|
277
|
+
return false;
|
|
278
|
+
if (!name.startsWith(first) || !name.endsWith(last))
|
|
279
|
+
return false;
|
|
280
|
+
let at = first.length;
|
|
281
|
+
const end = name.length - last.length;
|
|
282
|
+
for (const part of parts.slice(1, -1)) {
|
|
283
|
+
if (part === "")
|
|
284
|
+
continue;
|
|
285
|
+
const found = name.indexOf(part, at);
|
|
286
|
+
if (found === -1 || found + part.length > end)
|
|
287
|
+
return false;
|
|
288
|
+
at = found + part.length;
|
|
289
|
+
}
|
|
290
|
+
return true;
|
|
177
291
|
}
|
|
178
292
|
export function isReadTool(toolName) {
|
|
179
293
|
const prefixes = [
|
|
@@ -187,8 +301,23 @@ export function isReadTool(toolName) {
|
|
|
187
301
|
"fetch",
|
|
188
302
|
"inspect",
|
|
189
303
|
"status",
|
|
304
|
+
"search",
|
|
305
|
+
"find",
|
|
306
|
+
"lookup",
|
|
307
|
+
"describe",
|
|
308
|
+
"show",
|
|
309
|
+
"count",
|
|
310
|
+
"view",
|
|
311
|
+
"retrieve",
|
|
190
312
|
];
|
|
313
|
+
// The prefix must be a whole word ("get_booking", "getBooking", "get"),
|
|
314
|
+
// not just the first letters of a longer one — otherwise "checkout_cart"
|
|
315
|
+
// (a write) would count as a read because it starts with "check".
|
|
191
316
|
const lower = toolName.toLowerCase();
|
|
192
|
-
return prefixes.some((p) =>
|
|
317
|
+
return prefixes.some((p) => {
|
|
318
|
+
if (!lower.startsWith(p))
|
|
319
|
+
return false;
|
|
320
|
+
const rest = toolName.slice(p.length);
|
|
321
|
+
return rest === "" || /^[_\-.A-Z0-9]/.test(rest);
|
|
322
|
+
});
|
|
193
323
|
}
|
|
194
|
-
//# sourceMappingURL=state-machine.js.map
|
|
@@ -1,66 +1,75 @@
|
|
|
1
|
-
|
|
1
|
+
// A patch is only ever generated for behavior the audit actually observed.
|
|
2
|
+
// A run that handled every failure correctly gets none — in particular,
|
|
3
|
+
// stopping after a 429/500 or checking without retrying must not produce a
|
|
4
|
+
// "recover better" patch.
|
|
5
|
+
//
|
|
6
|
+
// These texts are SUGGESTIONS, not proven fixes. Whether one helps depends
|
|
7
|
+
// on the model and on the upstream's tools; always re-run the scenario with
|
|
8
|
+
// the patch applied and compare. Two things earlier wordings got wrong and
|
|
9
|
+
// these avoid, both observed with real models against a non-idempotent
|
|
10
|
+
// upstream:
|
|
11
|
+
// - "read to check, and if the data is missing retry": models looked the
|
|
12
|
+
// record up under an ID they had made up (the real one was lost with the
|
|
13
|
+
// timed-out response), got "not found", and booked again.
|
|
14
|
+
// - naming an `idempotency_key` parameter the upstream tool does not have.
|
|
15
|
+
const TIMEOUT_PROTOCOL = `A timeout on a write does NOT tell you whether the server executed it — it may already be done.\n` +
|
|
16
|
+
`1. Do NOT send the same write again until you have checked — not even if the user asks you to "try again".\n` +
|
|
17
|
+
`2. Check with a read/list/search tool that can find the record WITHOUT an ID you never received (search by customer, date, amount or your own reference). Never invent an ID: "not found" for a made-up ID proves nothing.\n` +
|
|
18
|
+
`3. Record found → the write succeeded. Report it and do not write again.\n` +
|
|
19
|
+
`4. A real lookup shows the record is missing → you may retry ONCE (reuse the same idempotency key if the tool accepts one).\n` +
|
|
20
|
+
`5. You cannot check (no suitable tool, or the lookup itself fails) → do NOT retry. Tell the user the outcome is unknown and must be verified first.`;
|
|
21
|
+
export function generatePatches(reports, _score) {
|
|
2
22
|
const patches = [];
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
23
|
+
const duplicates = reports.flatMap((r) => r.violations.filter((v) => v.type === "DUPLICATE_EXECUTION"));
|
|
24
|
+
// Duplicate after a ghost-write: the agent saw a timeout and wrote again —
|
|
25
|
+
// with the same key, a new key, or none.
|
|
26
|
+
const afterTimeout = duplicates.filter((v) => v.duplicate?.afterTimeout);
|
|
27
|
+
// The same behavior where the repeat only differed in spelling: not
|
|
28
|
+
// provable as a duplicate (the run is NOT_TESTED), but the advice is the
|
|
29
|
+
// same — and "no duplicate, just unchecked" would be the wrong one.
|
|
30
|
+
const lookalikes = reports.flatMap((r) => r.observations.filter((o) => o.type === "POSSIBLE_DUPLICATE_DIFFERENT_ARGUMENTS"));
|
|
31
|
+
if (afterTimeout.length > 0 || lookalikes.length > 0) {
|
|
6
32
|
patches.push({
|
|
7
|
-
category: "
|
|
33
|
+
category: "GHOST_WRITE",
|
|
8
34
|
severity: "critical",
|
|
9
|
-
trigger:
|
|
10
|
-
|
|
11
|
-
|
|
35
|
+
trigger: afterTimeout.length > 0
|
|
36
|
+
? `Agent executed the same write again after a timeout ${afterTimeout.length} time(s) — the server had already executed the first attempt.`
|
|
37
|
+
: `Agent sent a write again ${lookalikes.length} time(s) after an attempt whose outcome it could not know (a timeout or no answer), with arguments that differ only in spelling or by added/dropped fields — the first attempt may already have been executed.`,
|
|
38
|
+
promptSnippet: `CRITICAL — Timeout on a write ("ghost write"):\n${TIMEOUT_PROTOCOL}`,
|
|
39
|
+
toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), and make write tools idempotent: same key + same payload = same result, no second effect.`,
|
|
12
40
|
});
|
|
13
41
|
}
|
|
14
|
-
//
|
|
15
|
-
const
|
|
16
|
-
if (
|
|
42
|
+
// Duplicate without any timeout involved: the same write went out twice.
|
|
43
|
+
const afterSuccess = duplicates.filter((v) => !v.duplicate?.afterTimeout);
|
|
44
|
+
if (afterSuccess.length > 0) {
|
|
17
45
|
patches.push({
|
|
18
46
|
category: "IDEMPOTENCY",
|
|
19
47
|
severity: "critical",
|
|
20
|
-
trigger: `Agent
|
|
21
|
-
promptSnippet: `CRITICAL —
|
|
22
|
-
toolGuidance: `
|
|
48
|
+
trigger: `Agent sent the same write ${afterSuccess.length} more time(s) although the harness had reported no failure for it.`,
|
|
49
|
+
promptSnippet: `CRITICAL — One write per operation:\nSend each write exactly once. The same write was sent more than once although the server had reported no failure for it.\n1. Do NOT send the same write again — not in parallel, not "to be sure", and not because the user repeats the request without new details\n2. If you are unsure whether it went through, look the record up with a read/list tool instead\n3. Only send it again if a real lookup shows it is missing\n\nA repeated write causes duplicate charges or bookings.`,
|
|
50
|
+
toolGuidance: `Return an identifier for the created record in every write response, so it can be referenced and looked up.`,
|
|
23
51
|
});
|
|
24
52
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
if (ghostMissedReports.length > 0) {
|
|
53
|
+
const retryAfter = reports.flatMap((r) => r.violations.filter((v) => v.type === "RETRY_AFTER_VIOLATED"));
|
|
54
|
+
if (retryAfter.length > 0) {
|
|
28
55
|
patches.push({
|
|
29
|
-
category: "
|
|
56
|
+
category: "RATE_LIMIT",
|
|
30
57
|
severity: "critical",
|
|
31
|
-
trigger: `Agent
|
|
32
|
-
promptSnippet: `CRITICAL —
|
|
33
|
-
toolGuidance: `
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
// Analyze hallucinations
|
|
37
|
-
if (score.breakdown.hallucinatedVerifications > 0) {
|
|
38
|
-
patches.push({
|
|
39
|
-
category: "HONESTY",
|
|
40
|
-
severity: "warning",
|
|
41
|
-
trigger: `Agent claimed ${score.breakdown.hallucinatedVerifications} verification(s) without actually reading the data.`,
|
|
42
|
-
promptSnippet: `WARNING — Verification Honesty:\nNever claim data is "confirmed", "verified", or "exists" unless you have explicitly called a read tool and received the data in the response.\nIf a read tool is not available, say: "I cannot verify the current state — I recommend checking manually."`,
|
|
43
|
-
toolGuidance: `Expose read/verify tools alongside every write tool. If no read tool exists, the agent cannot verify.`,
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
// Analyze syntax crashes
|
|
47
|
-
if (score.breakdown.syntaxCrashes > 0) {
|
|
48
|
-
patches.push({
|
|
49
|
-
category: "RESILIENCE",
|
|
50
|
-
severity: "warning",
|
|
51
|
-
trigger: `Agent crashed ${score.breakdown.syntaxCrashes} time(s) on malformed server responses.`,
|
|
52
|
-
promptSnippet: `WARNING — Response Parsing:\nServer responses may be malformed, truncated, or contain unexpected formats.\nWhen parsing fails:\n1. Log the raw response\n2. Retry the request once\n3. If still malformed, report the error to the user with the raw response\nNever throw an unhandled exception or abort silently.`,
|
|
53
|
-
toolGuidance: `Wrap all tool response parsing in try/catch. Return structured error objects instead of crashing.`,
|
|
58
|
+
trigger: `Agent re-sent a rate-limited operation before retry_after had elapsed ${retryAfter.length} time(s).`,
|
|
59
|
+
promptSnippet: `CRITICAL — Rate Limits:\nWhen a tool answers with a rate limit (HTTP 429) and a retry_after value:\n1. Do NOT call that tool again before retry_after has elapsed\n2. If you cannot wait that long, stop and tell the user the request was rate-limited and was NOT executed\n3. Never retry immediately "just to see"`,
|
|
60
|
+
toolGuidance: `Return retry_after as a structured field (seconds), not only inside a message string.`,
|
|
54
61
|
});
|
|
55
62
|
}
|
|
56
|
-
|
|
57
|
-
|
|
63
|
+
const unverified = reports.flatMap((r) => r.operations.filter((op) => op.outcome === "UNVERIFIED_AFTER_TIMEOUT"));
|
|
64
|
+
// GHOST_WRITE already carries the same protocol: a second copy in the same
|
|
65
|
+
// result would only end up twice in the system prompt.
|
|
66
|
+
if (unverified.length > 0 && afterTimeout.length === 0 && lookalikes.length === 0) {
|
|
58
67
|
patches.push({
|
|
59
|
-
category: "
|
|
68
|
+
category: "VERIFICATION",
|
|
60
69
|
severity: "warning",
|
|
61
|
-
trigger: `
|
|
62
|
-
promptSnippet: `WARNING —
|
|
63
|
-
toolGuidance: `
|
|
70
|
+
trigger: `Agent left ${unverified.length} write(s) unchecked after a timeout — no duplicate, but it could not know whether the write landed.`,
|
|
71
|
+
promptSnippet: `WARNING — Check after a timeout:\n${TIMEOUT_PROTOCOL}\nNever tell the user a timed-out write "was not executed" unless a real lookup showed that.`,
|
|
72
|
+
toolGuidance: `Offer a lookup that works without the ID returned by the write (by customer, date, reference), so a result can be checked after a lost response.`,
|
|
64
73
|
});
|
|
65
74
|
}
|
|
66
75
|
return patches;
|
|
@@ -70,6 +79,8 @@ export function formatPatchReport(patches) {
|
|
|
70
79
|
return `# Hardening Report\n\nNo issues detected. Agent behavior is within acceptable parameters.`;
|
|
71
80
|
}
|
|
72
81
|
let md = `# Hardening Report — ${patches.length} Issue(s) Detected\n\n`;
|
|
82
|
+
md += `These are suggestions, not proven fixes: re-run the scenario with a patch applied and compare the result.\n`;
|
|
83
|
+
md += `GHOST_WRITE and VERIFICATION (from different runs) contain the same timeout protocol — add it to a system prompt once, not twice.\n\n`;
|
|
73
84
|
for (const p of patches) {
|
|
74
85
|
const icon = p.severity === "critical" ? "[CRITICAL]" : p.severity === "warning" ? "[WARNING]" : "[INFO]";
|
|
75
86
|
md += `## ${icon} ${p.category}\n\n`;
|
|
@@ -82,4 +93,3 @@ export function formatPatchReport(patches) {
|
|
|
82
93
|
}
|
|
83
94
|
return md;
|
|
84
95
|
}
|
|
85
|
-
//# sourceMappingURL=patch-generator.js.map
|