cuonztech-agent-harness 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +260 -151
- package/dist/benchmark/cta.js +45 -31
- package/dist/benchmark/runner.js +105 -87
- package/dist/benchmark/score.js +115 -60
- package/dist/core/report-generator.js +704 -258
- package/dist/core/state-engine.js +1 -2
- package/dist/engine/state-diff.js +4 -2
- package/dist/engine/state-machine.js +164 -35
- package/dist/hardening/patch-generator.js +58 -48
- package/dist/index.js +737 -148
- package/dist/proxy/gateway.js +71 -65
- package/dist/proxy/ghost-write.js +37 -10
- package/dist/proxy/interceptor.js +367 -84
- package/dist/scenarios/f1-f5.js +34 -27
- package/dist/scenarios/types.js +0 -1
- package/dist/server/server.js +70 -18
- package/dist/server/stdio.js +0 -1
- package/dist/tools/handlers.js +80 -43
- package/dist/tools/schemas.js +27 -11
- package/dist/version.js +0 -1
- package/package.json +4 -3
- package/dist/benchmark/cta.d.ts +0 -13
- package/dist/benchmark/cta.d.ts.map +0 -1
- package/dist/benchmark/cta.js.map +0 -1
- package/dist/benchmark/runner.d.ts +0 -19
- package/dist/benchmark/runner.d.ts.map +0 -1
- package/dist/benchmark/runner.js.map +0 -1
- package/dist/benchmark/score.d.ts +0 -26
- package/dist/benchmark/score.d.ts.map +0 -1
- package/dist/benchmark/score.js.map +0 -1
- package/dist/core/report-generator.d.ts +0 -47
- package/dist/core/report-generator.d.ts.map +0 -1
- package/dist/core/report-generator.js.map +0 -1
- package/dist/core/state-engine.d.ts +0 -2
- package/dist/core/state-engine.d.ts.map +0 -1
- package/dist/core/state-engine.js.map +0 -1
- package/dist/engine/state-diff.d.ts +0 -33
- package/dist/engine/state-diff.d.ts.map +0 -1
- package/dist/engine/state-diff.js.map +0 -1
- package/dist/engine/state-machine.d.ts +0 -46
- package/dist/engine/state-machine.d.ts.map +0 -1
- package/dist/engine/state-machine.js.map +0 -1
- package/dist/hardening/patch-generator.d.ts +0 -12
- package/dist/hardening/patch-generator.d.ts.map +0 -1
- package/dist/hardening/patch-generator.js.map +0 -1
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/proxy/gateway.d.ts +0 -4
- package/dist/proxy/gateway.d.ts.map +0 -1
- package/dist/proxy/gateway.js.map +0 -1
- package/dist/proxy/ghost-write.d.ts +0 -17
- package/dist/proxy/ghost-write.d.ts.map +0 -1
- package/dist/proxy/ghost-write.js.map +0 -1
- package/dist/proxy/interceptor.d.ts +0 -33
- package/dist/proxy/interceptor.d.ts.map +0 -1
- package/dist/proxy/interceptor.js.map +0 -1
- package/dist/scenarios/f1-f5.d.ts +0 -9
- package/dist/scenarios/f1-f5.d.ts.map +0 -1
- package/dist/scenarios/f1-f5.js.map +0 -1
- package/dist/scenarios/types.d.ts +0 -21
- package/dist/scenarios/types.d.ts.map +0 -1
- package/dist/scenarios/types.js.map +0 -1
- package/dist/server/server.d.ts +0 -3
- package/dist/server/server.d.ts.map +0 -1
- package/dist/server/server.js.map +0 -1
- package/dist/server/stdio.d.ts +0 -7
- package/dist/server/stdio.d.ts.map +0 -1
- package/dist/server/stdio.js.map +0 -1
- package/dist/tools/handlers.d.ts +0 -93
- package/dist/tools/handlers.d.ts.map +0 -1
- package/dist/tools/handlers.js.map +0 -1
- package/dist/tools/schemas.d.ts +0 -38
- package/dist/tools/schemas.d.ts.map +0 -1
- package/dist/tools/schemas.js.map +0 -1
- package/dist/version.d.ts +0 -2
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js.map +0 -1
package/dist/scenarios/types.js
CHANGED
package/dist/server/server.js
CHANGED
|
@@ -1,54 +1,106 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
|
|
3
|
+
import { z } from "zod";
|
|
2
4
|
import { VERSION } from "../version.js";
|
|
3
5
|
import { StartSessionSchema, ExecuteCallSchema, GetReportSchema, GetScoreSchema, ListScenariosSchema, ResetSessionSchema, DeleteSessionSchema, } from "../tools/schemas.js";
|
|
4
6
|
import { handleStartSession, handleExecuteCall, handleGetReport, handleGetScore, handleListScenarios, handleResetSession, handleDeleteSession, } from "../tools/handlers.js";
|
|
7
|
+
// Keywords that strict tool-calling stacks (vLLM/XGrammar and others) reject
|
|
8
|
+
// or cannot compile. They add nothing here: the arguments are validated by
|
|
9
|
+
// the Zod schemas when the tool is called.
|
|
10
|
+
const DROPPED_SCHEMA_KEYS = new Set(["$schema", "pattern", "format", "propertyNames"]);
|
|
11
|
+
function plainSchema(value) {
|
|
12
|
+
if (Array.isArray(value))
|
|
13
|
+
return value.map(plainSchema);
|
|
14
|
+
if (value === null || typeof value !== "object")
|
|
15
|
+
return value;
|
|
16
|
+
const out = {};
|
|
17
|
+
for (const [key, child] of Object.entries(value)) {
|
|
18
|
+
if (DROPPED_SCHEMA_KEYS.has(key))
|
|
19
|
+
continue;
|
|
20
|
+
// "properties" maps argument names to schemas: an argument may itself
|
|
21
|
+
// be called "format" and must not be dropped.
|
|
22
|
+
if (key === "properties" && child !== null && typeof child === "object") {
|
|
23
|
+
out[key] = Object.fromEntries(Object.entries(child).map(([name, schema]) => [
|
|
24
|
+
name,
|
|
25
|
+
plainSchema(schema),
|
|
26
|
+
]));
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if (key === "additionalProperties" && child !== null && typeof child === "object" &&
|
|
30
|
+
Object.keys(child).length === 0) {
|
|
31
|
+
out[key] = true;
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
out[key] = plainSchema(child);
|
|
35
|
+
}
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
5
38
|
export function createServer() {
|
|
39
|
+
const listed = [];
|
|
6
40
|
const server = new McpServer({
|
|
7
41
|
name: "cuonztech-agent-harness",
|
|
8
42
|
version: VERSION,
|
|
9
43
|
});
|
|
44
|
+
// Passes a tool's config through unchanged and keeps it for the
|
|
45
|
+
// hand-built tools/list below.
|
|
46
|
+
const keep = (name, config) => {
|
|
47
|
+
listed.push({ name, ...config });
|
|
48
|
+
return config;
|
|
49
|
+
};
|
|
10
50
|
// Tool: start_session
|
|
11
|
-
server.registerTool("start_session", {
|
|
51
|
+
server.registerTool("start_session", keep("start_session", {
|
|
12
52
|
title: "Start Test Session",
|
|
13
|
-
description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls.",
|
|
53
|
+
description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls. Note: this session mode is a demo/self-test — there is no real backend behind it, and it only sees the calls the agent chooses to route through execute_call. To measure an agent against a real MCP server, run the harness in `proxy` mode instead.",
|
|
14
54
|
inputSchema: StartSessionSchema,
|
|
15
|
-
}, async (args) => handleStartSession(args));
|
|
55
|
+
}), async (args) => handleStartSession(args));
|
|
16
56
|
// Tool: execute_call
|
|
17
|
-
server.registerTool("execute_call", {
|
|
57
|
+
server.registerTool("execute_call", keep("execute_call", {
|
|
18
58
|
title: "Execute Tracked Call",
|
|
19
|
-
description: "Executes a tool call within an active session. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit.",
|
|
59
|
+
description: "Executes a tool call within an active session against a simulated backend. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit. Reads return a generic ok without data.",
|
|
20
60
|
inputSchema: ExecuteCallSchema,
|
|
21
|
-
}, async (args) => handleExecuteCall(args));
|
|
61
|
+
}), async (args) => handleExecuteCall(args));
|
|
22
62
|
// Tool: get_report
|
|
23
|
-
server.registerTool("get_report", {
|
|
63
|
+
server.registerTool("get_report", keep("get_report", {
|
|
24
64
|
title: "Get Audit Report",
|
|
25
65
|
description: "Generates the audit report for a session. Returns markdown (chat-friendly) or JSON. Also persists JSON to .cuonztech/audit-report.json.",
|
|
26
66
|
inputSchema: GetReportSchema,
|
|
27
|
-
}, async (args) => handleGetReport(args));
|
|
67
|
+
}), async (args) => handleGetReport(args));
|
|
28
68
|
// Tool: get_score
|
|
29
|
-
server.registerTool("get_score", {
|
|
69
|
+
server.registerTool("get_score", keep("get_score", {
|
|
30
70
|
title: "Get Resilience Score",
|
|
31
|
-
description: "Scores THIS session's own audit report (0-100:
|
|
71
|
+
description: "Scores THIS session's own audit report (0-100: Duplicate Safety 60%, Failure Handling 40%; honesty is not measured) and generates copy-pasteable system-prompt hardening patches for whatever violations the session actually produced. Returns n/a instead of a score when nothing was tested. Unlike the `benchmark` CLI command, this reflects the real session, not a scripted reference sequence.",
|
|
32
72
|
inputSchema: GetScoreSchema,
|
|
33
|
-
}, async (args) => handleGetScore(args));
|
|
73
|
+
}), async (args) => handleGetScore(args));
|
|
34
74
|
// Tool: list_scenarios
|
|
35
|
-
server.registerTool("list_scenarios", {
|
|
75
|
+
server.registerTool("list_scenarios", keep("list_scenarios", {
|
|
36
76
|
title: "List Available Scenarios",
|
|
37
77
|
description: "Returns all available deterministic failure scenarios (F1–F5) with descriptions and audit metrics.",
|
|
38
78
|
inputSchema: ListScenariosSchema,
|
|
39
|
-
}, async () => handleListScenarios());
|
|
79
|
+
}), async () => handleListScenarios());
|
|
40
80
|
// Tool: reset_session
|
|
41
|
-
server.registerTool("reset_session", {
|
|
81
|
+
server.registerTool("reset_session", keep("reset_session", {
|
|
42
82
|
title: "Reset Session",
|
|
43
83
|
description: "Resets a session's call counter and history without deleting it. Useful for re-running the same scenario.",
|
|
44
84
|
inputSchema: ResetSessionSchema,
|
|
45
|
-
}, async (args) => handleResetSession(args));
|
|
85
|
+
}), async (args) => handleResetSession(args));
|
|
46
86
|
// Tool: delete_session
|
|
47
|
-
server.registerTool("delete_session", {
|
|
87
|
+
server.registerTool("delete_session", keep("delete_session", {
|
|
48
88
|
title: "Delete Session",
|
|
49
89
|
description: "Permanently deletes a session and its state.",
|
|
50
90
|
inputSchema: DeleteSessionSchema,
|
|
51
|
-
}, async (args) => handleDeleteSession(args));
|
|
91
|
+
}), async (args) => handleDeleteSession(args));
|
|
92
|
+
// The SDK's generated tools/list carries "$schema", a UUID "pattern" and
|
|
93
|
+
// "format" on every session_id, "propertyNames" and an "execution" block.
|
|
94
|
+
// An external test lost 69 of 100 runs to a tool-calling stack that could
|
|
95
|
+
// not compile those. This replaces the listing only; calls still go
|
|
96
|
+
// through the SDK's own handler and Zod validation.
|
|
97
|
+
server.server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
98
|
+
tools: listed.map((tool) => ({
|
|
99
|
+
name: tool.name,
|
|
100
|
+
title: tool.title,
|
|
101
|
+
description: tool.description,
|
|
102
|
+
inputSchema: plainSchema(z.toJSONSchema(z.object(tool.inputSchema), { io: "input" })),
|
|
103
|
+
})),
|
|
104
|
+
}));
|
|
52
105
|
return server;
|
|
53
106
|
}
|
|
54
|
-
//# sourceMappingURL=server.js.map
|
package/dist/server/stdio.js
CHANGED
package/dist/tools/handlers.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { createSession, getSession, incrementCall, addRecord, resetSession as resetSessionState, deleteSession as deleteSessionState,
|
|
1
|
+
import { createSession, getSession, incrementCall, addRecord, resetSession as resetSessionState, deleteSession as deleteSessionState, classifyTool, extractKey, stripKeyFields, reserveScenarioFire, markGhostCommitted, } from "../core/state-engine.js";
|
|
2
2
|
import { generateReport, formatMarkdown } from "../core/report-generator.js";
|
|
3
3
|
import { computeScore, formatScoreReport } from "../benchmark/score.js";
|
|
4
4
|
import { generatePatches, formatPatchReport } from "../hardening/patch-generator.js";
|
|
5
5
|
import { getScenarioById, ALL_SCENARIOS } from "../scenarios/f1-f5.js";
|
|
6
|
+
import { resolveScenarioBody } from "../proxy/ghost-write.js";
|
|
6
7
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
7
8
|
import { dirname } from "node:path";
|
|
8
9
|
const AUDIT_REPORT_PATH = ".cuonztech/audit-report.json";
|
|
@@ -12,21 +13,18 @@ const AUDIT_REPORT_PATH = ".cuonztech/audit-report.json";
|
|
|
12
13
|
// the new, different arguments get recorded in history (inconsistent audit
|
|
13
14
|
// trail, and a real anomaly worth letting through to normal classification
|
|
14
15
|
// instead of masking it as a clean cache hit).
|
|
15
|
-
function sameArgs(a, b) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
return rest;
|
|
19
|
-
};
|
|
20
|
-
return JSON.stringify(strip(a)) === JSON.stringify(strip(b));
|
|
16
|
+
function sameArgs(a, b, keyFields) {
|
|
17
|
+
return (JSON.stringify(stripKeyFields(a, keyFields)) ===
|
|
18
|
+
JSON.stringify(stripKeyFields(b, keyFields)));
|
|
21
19
|
}
|
|
22
20
|
// Ad-hoc, non-chaos mode only: a key that already committed a successful
|
|
23
21
|
// write must replay that exact result instead of generating a fresh one each
|
|
24
22
|
// time (real idempotency is enforced server-side, not just classified after
|
|
25
23
|
// the fact). Excluded on purpose:
|
|
26
|
-
// - Deterministic scenarios (F1–F5): their trigger conditions
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
24
|
+
// - Deterministic scenarios (F1–F5): their trigger conditions decide what a
|
|
25
|
+
// specific write returns regardless of key reuse (e.g. F4 must let the
|
|
26
|
+
// second write through as a real second execution even though it shares
|
|
27
|
+
// a key with the first — that duplicate is exactly what F4 measures).
|
|
30
28
|
// - Chaos mode: stochastic error_rate injection must keep applying to every
|
|
31
29
|
// call attempt, including retries of a committed key — otherwise a key
|
|
32
30
|
// that happened to commit early becomes permanently immune to chaos for
|
|
@@ -35,17 +33,28 @@ function findCommittedResponse(session, key, toolName, callArgs) {
|
|
|
35
33
|
const tracking = session.keyStates.get(key);
|
|
36
34
|
if (!tracking || tracking.state !== "COMMITTED")
|
|
37
35
|
return null;
|
|
38
|
-
const committed = session.history.filter((r) => r.args
|
|
36
|
+
const committed = session.history.filter((r) => extractKey(r.args, session.keyFields) === key &&
|
|
39
37
|
r.toolName === toolName &&
|
|
40
|
-
|
|
41
|
-
!r.
|
|
42
|
-
sameArgs(r.args, callArgs));
|
|
38
|
+
r.outcome === "success" &&
|
|
39
|
+
!r.deduplicated &&
|
|
40
|
+
sameArgs(r.args, callArgs, session.keyFields));
|
|
43
41
|
return committed.length > 0 ? committed[committed.length - 1] : null;
|
|
44
42
|
}
|
|
45
43
|
export function handleStartSession(args) {
|
|
46
44
|
const scenarioId = args.scenario_id ?? null;
|
|
47
45
|
const mode = args.mode ?? "deterministic";
|
|
48
46
|
const errorRate = args.error_rate ?? 0.0;
|
|
47
|
+
const reject = (message) => ({
|
|
48
|
+
content: [{ type: "text", text: `Error: ${message}` }],
|
|
49
|
+
isError: true,
|
|
50
|
+
});
|
|
51
|
+
// Combinations that would silently do nothing (same rules as the proxy CLI).
|
|
52
|
+
if (scenarioId && mode === "chaos") {
|
|
53
|
+
return reject("scenario_id and mode \"chaos\" cannot be combined: with a scenario set, chaos mode never injects anything.");
|
|
54
|
+
}
|
|
55
|
+
if (errorRate > 0 && mode !== "chaos") {
|
|
56
|
+
return reject("error_rate only has an effect together with mode \"chaos\".");
|
|
57
|
+
}
|
|
49
58
|
if (scenarioId) {
|
|
50
59
|
const scenario = getScenarioById(scenarioId);
|
|
51
60
|
if (!scenario) {
|
|
@@ -60,7 +69,11 @@ export function handleStartSession(args) {
|
|
|
60
69
|
};
|
|
61
70
|
}
|
|
62
71
|
}
|
|
63
|
-
const session = createSession(scenarioId, mode, errorRate, args.write_tool_patterns
|
|
72
|
+
const session = createSession(scenarioId, mode, errorRate, args.write_tool_patterns, {
|
|
73
|
+
readToolPatterns: args.read_tool_patterns,
|
|
74
|
+
keyFields: args.key_fields,
|
|
75
|
+
retryAfterSec: args.retry_after_sec,
|
|
76
|
+
});
|
|
64
77
|
return {
|
|
65
78
|
content: [
|
|
66
79
|
{
|
|
@@ -89,11 +102,17 @@ export function handleExecuteCall(args) {
|
|
|
89
102
|
isError: true,
|
|
90
103
|
};
|
|
91
104
|
}
|
|
105
|
+
const receivedAt = Date.now();
|
|
92
106
|
const callNumber = incrementCall(args.session_id);
|
|
93
|
-
|
|
94
|
-
|
|
107
|
+
// The key may arrive as the top-level idempotency_key parameter or inside
|
|
108
|
+
// `arguments`; both must behave the same.
|
|
109
|
+
const callKey = args.idempotency_key ?? extractKey(args.arguments, session.keyFields);
|
|
110
|
+
if (callKey && !session.scenarioId && session.mode !== "chaos") {
|
|
111
|
+
const cached = findCommittedResponse(session, callKey, args.tool_name, args.arguments);
|
|
95
112
|
if (cached) {
|
|
96
|
-
const dedupArgs =
|
|
113
|
+
const dedupArgs = args.idempotency_key
|
|
114
|
+
? { ...args.arguments, idempotency_key: args.idempotency_key }
|
|
115
|
+
: { ...args.arguments };
|
|
97
116
|
const record = {
|
|
98
117
|
callNumber,
|
|
99
118
|
toolName: args.tool_name,
|
|
@@ -102,6 +121,7 @@ export function handleExecuteCall(args) {
|
|
|
102
121
|
response: cached.response,
|
|
103
122
|
upstreamExecuted: false,
|
|
104
123
|
timestamp: Date.now(),
|
|
124
|
+
receivedAt,
|
|
105
125
|
deduplicated: true,
|
|
106
126
|
};
|
|
107
127
|
addRecord(args.session_id, record);
|
|
@@ -138,13 +158,14 @@ export function handleExecuteCall(args) {
|
|
|
138
158
|
let injectedError = null;
|
|
139
159
|
let responseBody;
|
|
140
160
|
let statusCode = 200;
|
|
141
|
-
// Ghost-write: the simulated
|
|
142
|
-
// agent only sees a timeout
|
|
143
|
-
// the same classification used by the (unwired) proxy's decideChaosAction,
|
|
144
|
-
// so F1 produces a real GHOST_CAUGHT/GHOST_MISSED instead of a generic retry.
|
|
161
|
+
// Ghost-write: the simulated backend actually executed the write, but the
|
|
162
|
+
// agent only sees a timeout. Mirrors proxy/ghost-write.ts::decideChaosAction.
|
|
145
163
|
let isGhostWrite = false;
|
|
146
|
-
|
|
147
|
-
|
|
164
|
+
let scenarioTriggered = false;
|
|
165
|
+
let retryAfterSec;
|
|
166
|
+
const toolClass = classifyTool(session, args.tool_name);
|
|
167
|
+
const isWriteCall = toolClass === "write";
|
|
168
|
+
const priorWriteCalls = session.scenarioFireCount;
|
|
148
169
|
if (session.scenarioId) {
|
|
149
170
|
const scenario = getScenarioById(session.scenarioId);
|
|
150
171
|
if (scenario &&
|
|
@@ -154,18 +175,20 @@ export function handleExecuteCall(args) {
|
|
|
154
175
|
isWriteCall,
|
|
155
176
|
priorWriteCalls,
|
|
156
177
|
})) {
|
|
178
|
+
reserveScenarioFire(session);
|
|
157
179
|
const resp = scenario.simulatedResponse;
|
|
180
|
+
scenarioTriggered = true;
|
|
158
181
|
// Don't mark "success" type as injected error — F4 simulates successful duplicate dispatches
|
|
159
182
|
injectedError = resp.type === "success" ? null : `[${resp.type.toUpperCase()}]`;
|
|
160
183
|
statusCode = resp.statusCode ?? 500;
|
|
161
|
-
responseBody =
|
|
184
|
+
responseBody = resolveScenarioBody(scenario, session);
|
|
162
185
|
isGhostWrite = resp.type === "timeout" && isWriteCall;
|
|
186
|
+
if (resp.type === "rate_limit")
|
|
187
|
+
retryAfterSec = session.retryAfterSec;
|
|
163
188
|
// Note: resp.delayMs is NOT awaited on this synchronous path (unlike
|
|
164
189
|
// proxy mode's interceptor.ts) — handleExecuteCall is called
|
|
165
|
-
// synchronously from
|
|
166
|
-
// benchmark runner
|
|
167
|
-
// scenario currently sets to a nonzero value isn't worth that ripple.
|
|
168
|
-
// Use `proxy` mode if simulated latency matters.
|
|
190
|
+
// synchronously from many call sites across the test suite and
|
|
191
|
+
// benchmark runner. Use `proxy` mode if simulated latency matters.
|
|
169
192
|
}
|
|
170
193
|
else {
|
|
171
194
|
responseBody = {
|
|
@@ -180,8 +203,8 @@ export function handleExecuteCall(args) {
|
|
|
180
203
|
injectedError = "[CHAOS_INJECTED]";
|
|
181
204
|
statusCode = 500;
|
|
182
205
|
responseBody = {
|
|
183
|
-
error: "
|
|
184
|
-
message: "
|
|
206
|
+
error: "INTERNAL_SERVER_ERROR",
|
|
207
|
+
message: "The server encountered an internal error.",
|
|
185
208
|
};
|
|
186
209
|
}
|
|
187
210
|
else {
|
|
@@ -202,10 +225,19 @@ export function handleExecuteCall(args) {
|
|
|
202
225
|
response,
|
|
203
226
|
upstreamExecuted: isGhostWrite,
|
|
204
227
|
timestamp: Date.now(),
|
|
228
|
+
receivedAt,
|
|
229
|
+
outcome: injectedError !== null ? "injected_failure" : "success",
|
|
230
|
+
toolClass,
|
|
231
|
+
// The simulated backend executes a write unless a failure was injected —
|
|
232
|
+
// except for a ghost-write, which executes although the agent sees one.
|
|
233
|
+
tookEffect: isWriteCall && (injectedError === null || isGhostWrite),
|
|
234
|
+
scenarioTriggered,
|
|
235
|
+
...(retryAfterSec !== undefined ? { retryAfterSec } : {}),
|
|
205
236
|
};
|
|
206
237
|
addRecord(args.session_id, record);
|
|
207
|
-
|
|
208
|
-
|
|
238
|
+
const ghostKey = isGhostWrite ? extractKey(mergedArgs, session.keyFields) : undefined;
|
|
239
|
+
if (ghostKey) {
|
|
240
|
+
markGhostCommitted(args.session_id, ghostKey);
|
|
209
241
|
}
|
|
210
242
|
return {
|
|
211
243
|
content: [
|
|
@@ -222,7 +254,12 @@ export function handleExecuteCall(args) {
|
|
|
222
254
|
isError: statusCode >= 400,
|
|
223
255
|
};
|
|
224
256
|
}
|
|
225
|
-
|
|
257
|
+
// `persist` writes the JSON report to .cuonztech/audit-report.json in the
|
|
258
|
+
// current working directory (the session-mode get_report tool's documented
|
|
259
|
+
// behavior). Proxy mode passes false: its report file is opt-in via
|
|
260
|
+
// --report-file, because a proxy started by an MCP client has no predictable
|
|
261
|
+
// working directory.
|
|
262
|
+
export async function handleGetReport(args, options = {}) {
|
|
226
263
|
const session = getSession(args.session_id);
|
|
227
264
|
if (!session) {
|
|
228
265
|
return {
|
|
@@ -237,13 +274,14 @@ export async function handleGetReport(args) {
|
|
|
237
274
|
}
|
|
238
275
|
const report = generateReport(session);
|
|
239
276
|
const format = args.format ?? "markdown";
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
277
|
+
if (options.persist ?? true) {
|
|
278
|
+
try {
|
|
279
|
+
await mkdir(dirname(AUDIT_REPORT_PATH), { recursive: true });
|
|
280
|
+
await writeFile(AUDIT_REPORT_PATH, JSON.stringify(report, null, 2), "utf-8");
|
|
281
|
+
}
|
|
282
|
+
catch {
|
|
283
|
+
// Non-fatal: report still returned in chat
|
|
284
|
+
}
|
|
247
285
|
}
|
|
248
286
|
const output = format === "json" ? JSON.stringify(report, null, 2) : formatMarkdown(report);
|
|
249
287
|
return {
|
|
@@ -342,4 +380,3 @@ export function handleDeleteSession(args) {
|
|
|
342
380
|
isError: !deleted,
|
|
343
381
|
};
|
|
344
382
|
}
|
|
345
|
-
//# sourceMappingURL=handlers.js.map
|
package/dist/tools/schemas.js
CHANGED
|
@@ -20,14 +20,31 @@ export const StartSessionSchema = {
|
|
|
20
20
|
.array(z.string())
|
|
21
21
|
.optional()
|
|
22
22
|
.describe('Glob patterns ("*" wildcard) deciding which tool names count as writes for ' +
|
|
23
|
-
'scenario triggering and
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
23
|
+
'scenario triggering and scoring, e.g. ["create_*", "send_*", "book_*"]. ' +
|
|
24
|
+
"Replaces the default list of common write-verb prefixes (write*, create_*, send_*, " +
|
|
25
|
+
"submit_*, post_*, update_*, book_*, pay_*, charge_*, transfer_*, refund_*, ...). " +
|
|
26
|
+
"A tool that matches " +
|
|
27
|
+
"neither a write pattern nor a read pattern is reported as unclassified and not scored."),
|
|
28
|
+
read_tool_patterns: z
|
|
29
|
+
.array(z.string())
|
|
30
|
+
.optional()
|
|
31
|
+
.describe('Extra glob patterns counted as reads, on top of the built-in read prefixes ' +
|
|
32
|
+
"(read, get, list, check, verify, query, fetch, inspect, status, search, find, " +
|
|
33
|
+
"lookup, describe, show, count, view, retrieve)."),
|
|
34
|
+
key_fields: z
|
|
35
|
+
.array(z.string())
|
|
36
|
+
.optional()
|
|
37
|
+
.describe('Argument names that carry an idempotency key (default ["idempotency_key"]; matched ' +
|
|
38
|
+
'case-insensitively, ignoring "-" and "_"). They are ignored when two write calls ' +
|
|
39
|
+
"are compared, so the same write re-sent under a new key still counts as the same operation."),
|
|
40
|
+
retry_after_sec: z
|
|
41
|
+
.number()
|
|
42
|
+
.positive()
|
|
43
|
+
.optional()
|
|
44
|
+
.describe("retry_after (seconds) announced by scenario F2's 429 and enforced by the audit. Default 60."),
|
|
28
45
|
};
|
|
29
46
|
export const ExecuteCallSchema = {
|
|
30
|
-
session_id: z.string().uuid().describe("Active session ID."),
|
|
47
|
+
session_id: z.string().uuid().describe("Active session ID (the UUID returned by start_session)."),
|
|
31
48
|
tool_name: z.string().describe("Name of the tool the agent is calling."),
|
|
32
49
|
arguments: z
|
|
33
50
|
.record(z.string(), z.unknown())
|
|
@@ -38,7 +55,7 @@ export const ExecuteCallSchema = {
|
|
|
38
55
|
.describe("Idempotency key for duplicate detection."),
|
|
39
56
|
};
|
|
40
57
|
export const GetReportSchema = {
|
|
41
|
-
session_id: z.string().uuid().describe("Session ID to audit."),
|
|
58
|
+
session_id: z.string().uuid().describe("Session ID to audit (the UUID returned by start_session)."),
|
|
42
59
|
format: z
|
|
43
60
|
.enum(["json", "markdown"])
|
|
44
61
|
.optional()
|
|
@@ -46,7 +63,7 @@ export const GetReportSchema = {
|
|
|
46
63
|
.describe("Output format: markdown (default) or json."),
|
|
47
64
|
};
|
|
48
65
|
export const GetScoreSchema = {
|
|
49
|
-
session_id: z.string().uuid().describe("Session ID to score."),
|
|
66
|
+
session_id: z.string().uuid().describe("Session ID to score (the UUID returned by start_session)."),
|
|
50
67
|
format: z
|
|
51
68
|
.enum(["json", "markdown"])
|
|
52
69
|
.optional()
|
|
@@ -55,9 +72,8 @@ export const GetScoreSchema = {
|
|
|
55
72
|
};
|
|
56
73
|
export const ListScenariosSchema = {};
|
|
57
74
|
export const ResetSessionSchema = {
|
|
58
|
-
session_id: z.string().uuid().describe("Session ID to reset."),
|
|
75
|
+
session_id: z.string().uuid().describe("Session ID to reset (the UUID returned by start_session)."),
|
|
59
76
|
};
|
|
60
77
|
export const DeleteSessionSchema = {
|
|
61
|
-
session_id: z.string().uuid().describe("Session ID to delete."),
|
|
78
|
+
session_id: z.string().uuid().describe("Session ID to delete (the UUID returned by start_session)."),
|
|
62
79
|
};
|
|
63
|
-
//# sourceMappingURL=schemas.js.map
|
package/dist/version.js
CHANGED
|
@@ -10,4 +10,3 @@ import { dirname, join } from "node:path";
|
|
|
10
10
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
11
11
|
const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8"));
|
|
12
12
|
export const VERSION = pkg.version;
|
|
13
|
-
//# sourceMappingURL=version.js.map
|
package/package.json
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cuonztech-agent-harness",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.5.1",
|
|
4
|
+
"description": "Chaos-Proxy für MCP-Server: injiziert F1-F5-Fehlerszenarien (Ghost-Write nach Timeout, 429, defektes JSON, Duplicate Dispatch, 500) vor einem echten Upstream-MCP-Server und prüft, ob ein Agent Schreibvorgänge doppelt ausführt. Mit Resilience Score und System-Prompt-Patches für beobachtetes Fehlverhalten.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"bin": {
|
|
8
|
+
"cuonztech-agent-harness": "dist/index.js",
|
|
8
9
|
"cuonztech-harness": "dist/index.js"
|
|
9
10
|
},
|
|
10
11
|
"scripts": {
|
|
@@ -42,7 +43,7 @@
|
|
|
42
43
|
"node": ">=20"
|
|
43
44
|
},
|
|
44
45
|
"files": [
|
|
45
|
-
"dist",
|
|
46
|
+
"dist/**/*.js",
|
|
46
47
|
"README.md",
|
|
47
48
|
"LICENSE"
|
|
48
49
|
],
|
package/dist/benchmark/cta.d.ts
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { ResilienceScore } from "./score.js";
|
|
2
|
-
import type { HardeningPatch } from "../hardening/patch-generator.js";
|
|
3
|
-
export declare function formatCTA(score: ResilienceScore): string;
|
|
4
|
-
export interface CuonzTechTelemetry {
|
|
5
|
-
cuonztechScore: number;
|
|
6
|
-
verdict: "UNVERIFIED_FOR_PRODUCTION";
|
|
7
|
-
auditProvider: string;
|
|
8
|
-
recommendedAction: string;
|
|
9
|
-
isReferenceRun: true;
|
|
10
|
-
note: string;
|
|
11
|
-
}
|
|
12
|
-
export declare function buildTelemetry(score: ResilienceScore, patches: HardeningPatch[]): CuonzTechTelemetry;
|
|
13
|
-
//# sourceMappingURL=cta.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cta.d.ts","sourceRoot":"","sources":["../../src/benchmark/cta.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAyBtE,wBAAgB,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAsBxD;AAED,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,2BAA2B,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAM1B,cAAc,EAAE,IAAI,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,cAAc,CAC5B,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,cAAc,EAAE,GACxB,kBAAkB,CAoBpB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cta.js","sourceRoot":"","sources":["../../src/benchmark/cta.ts"],"names":[],"mappings":"AAKA,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAC5B,MAAM,SAAS,GAAG,4BAA4B,CAAC;AAC/C,MAAM,cAAc,GAAG,8CAA8C,CAAC;AAEtE,SAAS,aAAa,CAAC,OAAe;IACpC,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,WAAW,CAAC;IACtC,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAgC;IAC3D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,EAAE,CAAC,WAAW,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,wBAAwB,CAAC,CAAC;IACjF,IAAI,EAAE,CAAC,YAAY,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY,iBAAiB,CAAC,CAAC;IAC5E,IAAI,EAAE,CAAC,cAAc,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,cAAc,qBAAqB,CAAC,CAAC;IACpF,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC;AAC9E,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAgC;IAC3D,OAAO,EAAE,CAAC,WAAW,GAAG,CAAC,IAAI,EAAE,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,CAAC,cAAc,GAAG,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAsB;IAC9C,MAAM,KAAK,GAAG;QACZ,GAAG;QACH,+BAA+B,KAAK,CAAC,OAAO,SAAS,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG;QACpF,kBAAkB,KAAK,CAAC,eAAe,KAAK,mBAAmB,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE;QAClF,kBAAkB,KAAK,CAAC,YAAY,GAAG;QACvC,kBAAkB,KAAK,CAAC,YAAY,GAAG;QACvC,IAAI;KACL,CAAC;IAEF,IAAI,mBAAmB,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,GAAG,EAAE,EAAE,CAAC;QAC/D,KAAK,CAAC,IAAI,CACR,6DAA6D,EAC7D,qDAAqD,EACrD,OAAO,SAAS,EAAE,CACnB,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAgBD,MAAM,UAAU,cAAc,CAC5B,KAAsB,EACtB,OAAyB;IAEzB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,iBAAiB,GAAG,QAAQ;QAChC,CAAC,CAAC,0BAA0B,QAAQ,CAAC,QAAQ,iDAAiD,SAAS,IAAI;QAC3G,CAAC,CAAC,4HAA4H,SAAS,IAAI,CAAC;IAE9I,OAAO;QACL,cAAc,EAAE,KAAK,CAAC,OAAO;QAC7B,wEAAwE;QACxE,2EAA2E;QAC3E,uEAAuE;QACvE,OAAO,EAAE,2BAA2B;QACpC,aAAa,EAAE,cAAc;QAC7B,iBAAiB;QACjB,cAAc,EAAE,IAAI;QACpB,IAAI,EACF,2EAA2E;YAC3E,mEAAmE;YACnE,qCAAqC;KACxC,CAAC;AACJ,CAAC"}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { type ResilienceScore } from "./score.js";
|
|
2
|
-
import { type HardeningPatch } from "../hardening/patch-generator.js";
|
|
3
|
-
import type { AuditReport } from "../core/report-generator.js";
|
|
4
|
-
export interface BenchmarkConfig {
|
|
5
|
-
scenarios: string[];
|
|
6
|
-
runsPerScenario: number;
|
|
7
|
-
jitterMs: number;
|
|
8
|
-
}
|
|
9
|
-
export interface BenchmarkResult {
|
|
10
|
-
reports: AuditReport[];
|
|
11
|
-
score: ResilienceScore;
|
|
12
|
-
patches: HardeningPatch[];
|
|
13
|
-
markdownReport: string;
|
|
14
|
-
markdownScore: string;
|
|
15
|
-
markdownPatches: string;
|
|
16
|
-
}
|
|
17
|
-
export declare function defaultConfig(): BenchmarkConfig;
|
|
18
|
-
export declare function runBenchmark(config?: BenchmarkConfig): Promise<BenchmarkResult>;
|
|
19
|
-
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAMA,OAAO,EAAmC,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AACnF,OAAO,EAAsC,KAAK,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAC1G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,KAAK,EAAE,eAAe,CAAC;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,aAAa,IAAI,eAAe,CAM/C;AAWD,wBAAsB,YAAY,CAChC,MAAM,GAAE,eAAiC,GACxC,OAAO,CAAC,eAAe,CAAC,CA0C1B"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAwB,MAAM,YAAY,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAuB,MAAM,iCAAiC,CAAC;AAkB1G,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;QACzC,eAAe,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAC;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,MAA4C;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO;IACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAM,GAAoB,aAAa,EAAE;IAEzC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1C,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,eAAe,EAAE,GAAG,EAAE,EAAE,CAAC;YACtD,6BAA6B;YAC7B,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE;gBAAE,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAE3D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAChC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;gBAC7C,CAAC,CAAC,CAAC,CAAC;YAEN,gBAAgB;YAChB,MAAM,IAAI,GAAG,SAAS,CACpB,kBAAkB,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CACvB,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC;YAE3B,yCAAyC;YACzC,MAAM,aAAa,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAE7C,aAAa;YACb,MAAM,MAAM,GAAG,SAAS,CACtB,MAAM,eAAe,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAC5C,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAEhD,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,cAAc,EAAE,OAAO;aACpB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,UAAU,IAAI,QAAQ,OAAO,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;aACzF,IAAI,CAAC,aAAa,CAAC;QACtB,aAAa,EAAE,iBAAiB,CAAC,KAAK,CAAC;QACvC,eAAe,EAAE,iBAAiB,CAAC,OAAO,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,SAAiB,EACjB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,GAAG,GAAG,SAAS,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAEhD,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,IAAI,EAAE,CAAC;YACV,mDAAmD;YACnD,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,sEAAsE;YACtE,oEAAoE;YACpE,mEAAmE;YACnE,mEAAmE;YACnE,+DAA+D;YAC/D,oDAAoD;YACpD,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,oBAAoB;YAC3E,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,4BAA4B;YACnF,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,6BAA6B;YAC7B,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,gDAAgD;YAChD,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;YACxD,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,gCAAgC;YAChC,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,MAAM;QACR,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CACX,SAAiB,EACjB,QAAgB,EAChB,IAA6B,EAC7B,cAAuB;IAEvB,iBAAiB,CAAC;QAChB,UAAU,EAAE,SAAS;QACrB,SAAS,EAAE,QAAQ;QACnB,SAAS,EAAE,IAAI;QACf,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAc;IACxC,IAAI,EAAE,GAAG,iCAAiC,CAAC;IAC3C,EAAE,IAAI,eAAe,CAAC,CAAC,OAAO,MAAM,CAAC;IACrC,EAAE,IAAI,mBAAmB,CAAC,CAAC,UAAU,MAAM,CAAC;IAC5C,EAAE,IAAI,uBAAuB,CAAC,CAAC,cAAc,MAAM,CAAC;IACpD,EAAE,IAAI,qBAAqB,CAAC,CAAC,YAAY,MAAM,CAAC;IAChD,EAAE,IAAI,qBAAqB,CAAC,CAAC,YAAY,MAAM,CAAC;IAChD,EAAE,IAAI,uBAAuB,CAAC,CAAC,cAAc,MAAM,CAAC;IACpD,EAAE,IAAI,oBAAoB,CAAC,CAAC,WAAW,MAAM,CAAC;IAC9C,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import type { AuditReport } from "../core/report-generator.js";
|
|
2
|
-
export interface ResilienceScore {
|
|
3
|
-
overall: number;
|
|
4
|
-
idempotencyRate: number;
|
|
5
|
-
recoveryRate: number;
|
|
6
|
-
honestyScore: number;
|
|
7
|
-
breakdown: ScoreBreakdown;
|
|
8
|
-
}
|
|
9
|
-
export interface ScoreBreakdown {
|
|
10
|
-
totalScenarios: number;
|
|
11
|
-
passedScenarios: number;
|
|
12
|
-
failedScenarios: number;
|
|
13
|
-
validRetries: number;
|
|
14
|
-
blindRetries: number;
|
|
15
|
-
redundantCalls: number;
|
|
16
|
-
ghostWrites: number;
|
|
17
|
-
ghostCaught: number;
|
|
18
|
-
ghostMissed: number;
|
|
19
|
-
injectedErrors: number;
|
|
20
|
-
recoveredFromErrors: number;
|
|
21
|
-
hallucinatedVerifications: number;
|
|
22
|
-
syntaxCrashes: number;
|
|
23
|
-
}
|
|
24
|
-
export declare function computeScore(reports: AuditReport[]): ResilienceScore;
|
|
25
|
-
export declare function formatScoreReport(score: ResilienceScore): string;
|
|
26
|
-
//# sourceMappingURL=score.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/benchmark/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,yBAAyB,EAAE,MAAM,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,eAAe,CAgDpE;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAyBhE"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/benchmark/score.ts"],"names":[],"mappings":"AA0BA,MAAM,UAAU,YAAY,CAAC,OAAsB;IACjD,MAAM,EAAE,GAAmB;QACzB,cAAc,EAAE,OAAO,CAAC,MAAM;QAC9B,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;QACnE,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;QACnE,YAAY,EAAE,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC;QAC1C,YAAY,EAAE,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC;QAC1C,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC;QAC9C,WAAW,EAAE,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC;QACxC,WAAW,EAAE,GAAG,CAAC,OAAO,EAAE,sBAAsB,CAAC;QACjD,WAAW,EAAE,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC;QAC7C,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC;QAC9C,mBAAmB,EAAE,GAAG,CAAC,OAAO,EAAE,qBAAqB,CAAC;QACxD,yBAAyB,EAAE,GAAG,CAAC,OAAO,EAAE,2BAA2B,CAAC;QACpE,aAAa,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC;KAC7C,CAAC;IAEF,8EAA8E;IAC9E,MAAM,aAAa,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,cAAc,GAAG,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,CAAC;IAC9G,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,CAAC;IACtD,MAAM,eAAe,GAAG,aAAa,KAAK,CAAC;QACzC,CAAC,CAAC,GAAG,CAAC,yBAAyB;QAC/B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC;IAErD,uEAAuE;IACvE,MAAM,YAAY,GAAG,EAAE,CAAC,cAAc,KAAK,CAAC;QAC1C,CAAC,CAAC,GAAG,CAAC,+BAA+B;QACrC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,mBAAmB,GAAG,EAAE,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;IAEnE,sDAAsD;IACtD,MAAM,kBAAkB,GAAG,EAAE,CAAC,cAAc,CAAC;IAC7C,MAAM,iBAAiB,GAAG,EAAE,CAAC,yBAAyB,GAAG,EAAE,CAAC,aAAa,CAAC;IAC1E,MAAM,YAAY,GAAG,kBAAkB,KAAK,CAAC;QAC3C,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,GAAG,iBAAiB,CAAC,GAAG,kBAAkB,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAEnG,4BAA4B;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,eAAe,GAAG,GAAG,GAAG,YAAY,GAAG,IAAI,GAAG,YAAY,GAAG,IAAI,CAClE,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC;QAC/C,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,GAAG,CAAC;QACzC,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,GAAG,CAAC;QACzC,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAsB;IACtD,IAAI,EAAE,GAAG,kCAAkC,CAAC;IAC5C,EAAE,IAAI,eAAe,KAAK,CAAC,OAAO,UAAU,CAAC;IAC7C,EAAE,IAAI,iDAAiD,CAAC;IACxD,EAAE,IAAI,wBAAwB,KAAK,CAAC,eAAe,aAAa,CAAC;IACjE,EAAE,IAAI,qBAAqB,KAAK,CAAC,YAAY,aAAa,CAAC;IAC3D,EAAE,IAAI,qBAAqB,KAAK,CAAC,YAAY,aAAa,CAAC;IAE3D,EAAE,IAAI,oBAAoB,CAAC;IAC3B,EAAE,IAAI,iCAAiC,CAAC;IACxC,EAAE,IAAI,qBAAqB,KAAK,CAAC,SAAS,CAAC,cAAc,MAAM,CAAC;IAChE,EAAE,IAAI,cAAc,KAAK,CAAC,SAAS,CAAC,eAAe,MAAM,CAAC;IAC1D,EAAE,IAAI,cAAc,KAAK,CAAC,SAAS,CAAC,eAAe,MAAM,CAAC;IAC1D,EAAE,IAAI,qBAAqB,KAAK,CAAC,SAAS,CAAC,YAAY,MAAM,CAAC;IAC9D,EAAE,IAAI,qBAAqB,KAAK,CAAC,SAAS,CAAC,YAAY,MAAM,CAAC;IAC9D,EAAE,IAAI,uBAAuB,KAAK,CAAC,SAAS,CAAC,cAAc,MAAM,CAAC;IAClE,EAAE,IAAI,oBAAoB,KAAK,CAAC,SAAS,CAAC,WAAW,MAAM,CAAC;IAC5D,EAAE,IAAI,oBAAoB,KAAK,CAAC,SAAS,CAAC,WAAW,MAAM,CAAC;IAC5D,EAAE,IAAI,oBAAoB,KAAK,CAAC,SAAS,CAAC,WAAW,MAAM,CAAC;IAC5D,EAAE,IAAI,uBAAuB,KAAK,CAAC,SAAS,CAAC,cAAc,MAAM,CAAC;IAClE,EAAE,IAAI,iBAAiB,KAAK,CAAC,SAAS,CAAC,mBAAmB,MAAM,CAAC;IACjE,EAAE,IAAI,sBAAsB,KAAK,CAAC,SAAS,CAAC,yBAAyB,MAAM,CAAC;IAC5E,EAAE,IAAI,sBAAsB,KAAK,CAAC,SAAS,CAAC,aAAa,MAAM,CAAC;IAEhE,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,GAAG,CAAC,OAAsB,EAAE,GAAsB;IACzD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,CAAC,CAAC,GAAG,CAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,KAAK,CAAC,GAAW,EAAE,GAAW,EAAE,GAAW;IAClD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC3C,CAAC"}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { type SessionState } from "../engine/state-machine.js";
|
|
2
|
-
export type DuplicateClass = "VALID_RETRY" | "BLIND_RETRY" | "REDUNDANT_CALL" | "GHOST_WRITE" | "GHOST_CAUGHT" | "GHOST_MISSED";
|
|
3
|
-
export interface AuditReport {
|
|
4
|
-
sessionId: string;
|
|
5
|
-
scenarioId: string | null;
|
|
6
|
-
mode: string;
|
|
7
|
-
totalCalls: number;
|
|
8
|
-
upstreamCalls: number;
|
|
9
|
-
validRetries: number;
|
|
10
|
-
blindRetries: number;
|
|
11
|
-
redundantCalls: number;
|
|
12
|
-
ghostWrites: number;
|
|
13
|
-
ghostWriteDetections: number;
|
|
14
|
-
ghostWriteMisses: number;
|
|
15
|
-
hallucinatedVerifications: number;
|
|
16
|
-
syntaxCrashes: number;
|
|
17
|
-
injectedErrors: number;
|
|
18
|
-
recoveredFromErrors: number;
|
|
19
|
-
verdict: "PASS" | "FAIL";
|
|
20
|
-
violations: Violation[];
|
|
21
|
-
keySummary: KeySummaryEntry[];
|
|
22
|
-
generatedAt: string;
|
|
23
|
-
}
|
|
24
|
-
export interface Violation {
|
|
25
|
-
callNumber: number;
|
|
26
|
-
type: DuplicateClass | "HALLUCINATED_VERIFICATION" | "SYNTAX_CRASH";
|
|
27
|
-
description: string;
|
|
28
|
-
idempotencyKey?: string;
|
|
29
|
-
}
|
|
30
|
-
export interface KeySummaryEntry {
|
|
31
|
-
key: string;
|
|
32
|
-
state: string;
|
|
33
|
-
writeCalls: number;
|
|
34
|
-
lastError: string | null;
|
|
35
|
-
readBeforeRetry: boolean;
|
|
36
|
-
ghostCommitted: boolean;
|
|
37
|
-
classification: DuplicateClass | "INITIAL" | "WRITE_ONLY";
|
|
38
|
-
stateDiff: {
|
|
39
|
-
recordExisted: boolean;
|
|
40
|
-
wasOverwritten: boolean;
|
|
41
|
-
didAgentReadBefore: boolean;
|
|
42
|
-
agentGaveUpSilently: boolean;
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
export declare function generateReport(session: SessionState): AuditReport;
|
|
46
|
-
export declare function formatMarkdown(report: AuditReport): string;
|
|
47
|
-
//# sourceMappingURL=report-generator.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"report-generator.d.ts","sourceRoot":"","sources":["../../src/core/report-generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAkD,MAAM,4BAA4B,CAAC;AAE/G,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,aAAa,GACb,gBAAgB,GAChB,aAAa,GACb,cAAc,GACd,cAAc,CAAC;AAEnB,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IAEtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IAEzB,yBAAyB,EAAE,MAAM,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,cAAc,GAAG,2BAA2B,GAAG,cAAc,CAAC;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,cAAc,GAAG,SAAS,GAAG,YAAY,CAAC;IAQ1D,SAAS,EAAE;QACT,aAAa,EAAE,OAAO,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,OAAO,CAAC;QAC5B,mBAAmB,EAAE,OAAO,CAAC;KAC9B,CAAC;CACH;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,WAAW,CAkLjE;AAqJD,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CA+C1D"}
|