cuonztech-agent-harness 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +260 -151
  2. package/dist/benchmark/cta.js +45 -31
  3. package/dist/benchmark/runner.js +105 -87
  4. package/dist/benchmark/score.js +115 -60
  5. package/dist/core/report-generator.js +704 -258
  6. package/dist/core/state-engine.js +1 -2
  7. package/dist/engine/state-diff.js +4 -2
  8. package/dist/engine/state-machine.js +164 -35
  9. package/dist/hardening/patch-generator.js +58 -48
  10. package/dist/index.js +737 -148
  11. package/dist/proxy/gateway.js +71 -65
  12. package/dist/proxy/ghost-write.js +37 -10
  13. package/dist/proxy/interceptor.js +367 -84
  14. package/dist/scenarios/f1-f5.js +34 -27
  15. package/dist/scenarios/types.js +0 -1
  16. package/dist/server/server.js +70 -18
  17. package/dist/server/stdio.js +0 -1
  18. package/dist/tools/handlers.js +80 -43
  19. package/dist/tools/schemas.js +27 -11
  20. package/dist/version.js +0 -1
  21. package/package.json +4 -3
  22. package/dist/benchmark/cta.d.ts +0 -13
  23. package/dist/benchmark/cta.d.ts.map +0 -1
  24. package/dist/benchmark/cta.js.map +0 -1
  25. package/dist/benchmark/runner.d.ts +0 -19
  26. package/dist/benchmark/runner.d.ts.map +0 -1
  27. package/dist/benchmark/runner.js.map +0 -1
  28. package/dist/benchmark/score.d.ts +0 -26
  29. package/dist/benchmark/score.d.ts.map +0 -1
  30. package/dist/benchmark/score.js.map +0 -1
  31. package/dist/core/report-generator.d.ts +0 -47
  32. package/dist/core/report-generator.d.ts.map +0 -1
  33. package/dist/core/report-generator.js.map +0 -1
  34. package/dist/core/state-engine.d.ts +0 -2
  35. package/dist/core/state-engine.d.ts.map +0 -1
  36. package/dist/core/state-engine.js.map +0 -1
  37. package/dist/engine/state-diff.d.ts +0 -33
  38. package/dist/engine/state-diff.d.ts.map +0 -1
  39. package/dist/engine/state-diff.js.map +0 -1
  40. package/dist/engine/state-machine.d.ts +0 -46
  41. package/dist/engine/state-machine.d.ts.map +0 -1
  42. package/dist/engine/state-machine.js.map +0 -1
  43. package/dist/hardening/patch-generator.d.ts +0 -12
  44. package/dist/hardening/patch-generator.d.ts.map +0 -1
  45. package/dist/hardening/patch-generator.js.map +0 -1
  46. package/dist/index.d.ts +0 -3
  47. package/dist/index.d.ts.map +0 -1
  48. package/dist/index.js.map +0 -1
  49. package/dist/proxy/gateway.d.ts +0 -4
  50. package/dist/proxy/gateway.d.ts.map +0 -1
  51. package/dist/proxy/gateway.js.map +0 -1
  52. package/dist/proxy/ghost-write.d.ts +0 -17
  53. package/dist/proxy/ghost-write.d.ts.map +0 -1
  54. package/dist/proxy/ghost-write.js.map +0 -1
  55. package/dist/proxy/interceptor.d.ts +0 -33
  56. package/dist/proxy/interceptor.d.ts.map +0 -1
  57. package/dist/proxy/interceptor.js.map +0 -1
  58. package/dist/scenarios/f1-f5.d.ts +0 -9
  59. package/dist/scenarios/f1-f5.d.ts.map +0 -1
  60. package/dist/scenarios/f1-f5.js.map +0 -1
  61. package/dist/scenarios/types.d.ts +0 -21
  62. package/dist/scenarios/types.d.ts.map +0 -1
  63. package/dist/scenarios/types.js.map +0 -1
  64. package/dist/server/server.d.ts +0 -3
  65. package/dist/server/server.d.ts.map +0 -1
  66. package/dist/server/server.js.map +0 -1
  67. package/dist/server/stdio.d.ts +0 -7
  68. package/dist/server/stdio.d.ts.map +0 -1
  69. package/dist/server/stdio.js.map +0 -1
  70. package/dist/tools/handlers.d.ts +0 -93
  71. package/dist/tools/handlers.d.ts.map +0 -1
  72. package/dist/tools/handlers.js.map +0 -1
  73. package/dist/tools/schemas.d.ts +0 -38
  74. package/dist/tools/schemas.d.ts.map +0 -1
  75. package/dist/tools/schemas.js.map +0 -1
  76. package/dist/version.d.ts +0 -2
  77. package/dist/version.d.ts.map +0 -1
  78. package/dist/version.js.map +0 -1
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=types.js.map
@@ -1,54 +1,106 @@
1
1
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
+ import { ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
3
+ import { z } from "zod";
2
4
  import { VERSION } from "../version.js";
3
5
  import { StartSessionSchema, ExecuteCallSchema, GetReportSchema, GetScoreSchema, ListScenariosSchema, ResetSessionSchema, DeleteSessionSchema, } from "../tools/schemas.js";
4
6
  import { handleStartSession, handleExecuteCall, handleGetReport, handleGetScore, handleListScenarios, handleResetSession, handleDeleteSession, } from "../tools/handlers.js";
7
+ // Keywords that strict tool-calling stacks (vLLM/XGrammar and others) reject
8
+ // or cannot compile. They add nothing here: the arguments are validated by
9
+ // the Zod schemas when the tool is called.
10
+ const DROPPED_SCHEMA_KEYS = new Set(["$schema", "pattern", "format", "propertyNames"]);
11
+ function plainSchema(value) {
12
+ if (Array.isArray(value))
13
+ return value.map(plainSchema);
14
+ if (value === null || typeof value !== "object")
15
+ return value;
16
+ const out = {};
17
+ for (const [key, child] of Object.entries(value)) {
18
+ if (DROPPED_SCHEMA_KEYS.has(key))
19
+ continue;
20
+ // "properties" maps argument names to schemas: an argument may itself
21
+ // be called "format" and must not be dropped.
22
+ if (key === "properties" && child !== null && typeof child === "object") {
23
+ out[key] = Object.fromEntries(Object.entries(child).map(([name, schema]) => [
24
+ name,
25
+ plainSchema(schema),
26
+ ]));
27
+ continue;
28
+ }
29
+ if (key === "additionalProperties" && child !== null && typeof child === "object" &&
30
+ Object.keys(child).length === 0) {
31
+ out[key] = true;
32
+ continue;
33
+ }
34
+ out[key] = plainSchema(child);
35
+ }
36
+ return out;
37
+ }
5
38
  export function createServer() {
39
+ const listed = [];
6
40
  const server = new McpServer({
7
41
  name: "cuonztech-agent-harness",
8
42
  version: VERSION,
9
43
  });
44
+ // Passes a tool's config through unchanged and keeps it for the
45
+ // hand-built tools/list below.
46
+ const keep = (name, config) => {
47
+ listed.push({ name, ...config });
48
+ return config;
49
+ };
10
50
  // Tool: start_session
11
- server.registerTool("start_session", {
51
+ server.registerTool("start_session", keep("start_session", {
12
52
  title: "Start Test Session",
13
- description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls.",
53
+ description: "Creates an isolated test session with optional scenario (F1–F5) and mode (deterministic/chaos). Returns a sessionId for subsequent calls. Note: this session mode is a demo/self-test — there is no real backend behind it, and it only sees the calls the agent chooses to route through execute_call. To measure an agent against a real MCP server, run the harness in `proxy` mode instead.",
14
54
  inputSchema: StartSessionSchema,
15
- }, async (args) => handleStartSession(args));
55
+ }), async (args) => handleStartSession(args));
16
56
  // Tool: execute_call
17
- server.registerTool("execute_call", {
57
+ server.registerTool("execute_call", keep("execute_call", {
18
58
  title: "Execute Tracked Call",
19
- description: "Executes a tool call within an active session. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit.",
59
+ description: "Executes a tool call within an active session against a simulated backend. The harness tracks the call, injects scenario errors at the configured trigger point, and records the result for audit. Reads return a generic ok without data.",
20
60
  inputSchema: ExecuteCallSchema,
21
- }, async (args) => handleExecuteCall(args));
61
+ }), async (args) => handleExecuteCall(args));
22
62
  // Tool: get_report
23
- server.registerTool("get_report", {
63
+ server.registerTool("get_report", keep("get_report", {
24
64
  title: "Get Audit Report",
25
65
  description: "Generates the audit report for a session. Returns markdown (chat-friendly) or JSON. Also persists JSON to .cuonztech/audit-report.json.",
26
66
  inputSchema: GetReportSchema,
27
- }, async (args) => handleGetReport(args));
67
+ }), async (args) => handleGetReport(args));
28
68
  // Tool: get_score
29
- server.registerTool("get_score", {
69
+ server.registerTool("get_score", keep("get_score", {
30
70
  title: "Get Resilience Score",
31
- description: "Scores THIS session's own audit report (0-100: Idempotency 40%, Recovery 35%, Honesty 25%) and generates copy-pasteable system-prompt hardening patches for whatever violations the connected agent actually produced. Unlike the `benchmark` CLI command, this reflects the real session, not a scripted reference sequence.",
71
+ description: "Scores THIS session's own audit report (0-100: Duplicate Safety 60%, Failure Handling 40%; honesty is not measured) and generates copy-pasteable system-prompt hardening patches for whatever violations the session actually produced. Returns n/a instead of a score when nothing was tested. Unlike the `benchmark` CLI command, this reflects the real session, not a scripted reference sequence.",
32
72
  inputSchema: GetScoreSchema,
33
- }, async (args) => handleGetScore(args));
73
+ }), async (args) => handleGetScore(args));
34
74
  // Tool: list_scenarios
35
- server.registerTool("list_scenarios", {
75
+ server.registerTool("list_scenarios", keep("list_scenarios", {
36
76
  title: "List Available Scenarios",
37
77
  description: "Returns all available deterministic failure scenarios (F1–F5) with descriptions and audit metrics.",
38
78
  inputSchema: ListScenariosSchema,
39
- }, async () => handleListScenarios());
79
+ }), async () => handleListScenarios());
40
80
  // Tool: reset_session
41
- server.registerTool("reset_session", {
81
+ server.registerTool("reset_session", keep("reset_session", {
42
82
  title: "Reset Session",
43
83
  description: "Resets a session's call counter and history without deleting it. Useful for re-running the same scenario.",
44
84
  inputSchema: ResetSessionSchema,
45
- }, async (args) => handleResetSession(args));
85
+ }), async (args) => handleResetSession(args));
46
86
  // Tool: delete_session
47
- server.registerTool("delete_session", {
87
+ server.registerTool("delete_session", keep("delete_session", {
48
88
  title: "Delete Session",
49
89
  description: "Permanently deletes a session and its state.",
50
90
  inputSchema: DeleteSessionSchema,
51
- }, async (args) => handleDeleteSession(args));
91
+ }), async (args) => handleDeleteSession(args));
92
+ // The SDK's generated tools/list carries "$schema", a UUID "pattern" and
93
+ // "format" on every session_id, "propertyNames" and an "execution" block.
94
+ // An external test lost 69 of 100 runs to a tool-calling stack that could
95
+ // not compile those. This replaces the listing only; calls still go
96
+ // through the SDK's own handler and Zod validation.
97
+ server.server.setRequestHandler(ListToolsRequestSchema, async () => ({
98
+ tools: listed.map((tool) => ({
99
+ name: tool.name,
100
+ title: tool.title,
101
+ description: tool.description,
102
+ inputSchema: plainSchema(z.toJSONSchema(z.object(tool.inputSchema), { io: "input" })),
103
+ })),
104
+ }));
52
105
  return server;
53
106
  }
54
- //# sourceMappingURL=server.js.map
@@ -3,4 +3,3 @@ export async function startStdio(server) {
3
3
  const transport = new StdioServerTransport();
4
4
  await server.connect(transport);
5
5
  }
6
- //# sourceMappingURL=stdio.js.map
@@ -1,8 +1,9 @@
1
- import { createSession, getSession, incrementCall, addRecord, resetSession as resetSessionState, deleteSession as deleteSessionState, isWriteTool, markGhostCommitted, } from "../core/state-engine.js";
1
+ import { createSession, getSession, incrementCall, addRecord, resetSession as resetSessionState, deleteSession as deleteSessionState, classifyTool, extractKey, stripKeyFields, reserveScenarioFire, markGhostCommitted, } from "../core/state-engine.js";
2
2
  import { generateReport, formatMarkdown } from "../core/report-generator.js";
3
3
  import { computeScore, formatScoreReport } from "../benchmark/score.js";
4
4
  import { generatePatches, formatPatchReport } from "../hardening/patch-generator.js";
5
5
  import { getScenarioById, ALL_SCENARIOS } from "../scenarios/f1-f5.js";
6
+ import { resolveScenarioBody } from "../proxy/ghost-write.js";
6
7
  import { mkdir, writeFile } from "node:fs/promises";
7
8
  import { dirname } from "node:path";
8
9
  const AUDIT_REPORT_PATH = ".cuonztech/audit-report.json";
@@ -12,21 +13,18 @@ const AUDIT_REPORT_PATH = ".cuonztech/audit-report.json";
12
13
  // the new, different arguments get recorded in history (inconsistent audit
13
14
  // trail, and a real anomaly worth letting through to normal classification
14
15
  // instead of masking it as a clean cache hit).
15
- function sameArgs(a, b) {
16
- const strip = (o) => {
17
- const { idempotency_key, ...rest } = o;
18
- return rest;
19
- };
20
- return JSON.stringify(strip(a)) === JSON.stringify(strip(b));
16
+ function sameArgs(a, b, keyFields) {
17
+ return (JSON.stringify(stripKeyFields(a, keyFields)) ===
18
+ JSON.stringify(stripKeyFields(b, keyFields)));
21
19
  }
22
20
  // Ad-hoc, non-chaos mode only: a key that already committed a successful
23
21
  // write must replay that exact result instead of generating a fresh one each
24
22
  // time (real idempotency is enforced server-side, not just classified after
25
23
  // the fact). Excluded on purpose:
26
- // - Deterministic scenarios (F1–F5): their trigger conditions are positional
27
- // (by callNumber) and intentionally decide what a specific call number
28
- // returns regardless of key reuse (e.g. F2 must still 429 the 2nd call
29
- // even though it shares a key with the 1st).
24
+ // - Deterministic scenarios (F1–F5): their trigger conditions decide what a
25
+ // specific write returns regardless of key reuse (e.g. F4 must let the
26
+ // second write through as a real second execution even though it shares
27
+ // a key with the first — that duplicate is exactly what F4 measures).
30
28
  // - Chaos mode: stochastic error_rate injection must keep applying to every
31
29
  // call attempt, including retries of a committed key — otherwise a key
32
30
  // that happened to commit early becomes permanently immune to chaos for
@@ -35,17 +33,28 @@ function findCommittedResponse(session, key, toolName, callArgs) {
35
33
  const tracking = session.keyStates.get(key);
36
34
  if (!tracking || tracking.state !== "COMMITTED")
37
35
  return null;
38
- const committed = session.history.filter((r) => r.args["idempotency_key"] === key &&
36
+ const committed = session.history.filter((r) => extractKey(r.args, session.keyFields) === key &&
39
37
  r.toolName === toolName &&
40
- !r.injectedError &&
41
- !r.response.includes('"error"') &&
42
- sameArgs(r.args, callArgs));
38
+ r.outcome === "success" &&
39
+ !r.deduplicated &&
40
+ sameArgs(r.args, callArgs, session.keyFields));
43
41
  return committed.length > 0 ? committed[committed.length - 1] : null;
44
42
  }
45
43
  export function handleStartSession(args) {
46
44
  const scenarioId = args.scenario_id ?? null;
47
45
  const mode = args.mode ?? "deterministic";
48
46
  const errorRate = args.error_rate ?? 0.0;
47
+ const reject = (message) => ({
48
+ content: [{ type: "text", text: `Error: ${message}` }],
49
+ isError: true,
50
+ });
51
+ // Combinations that would silently do nothing (same rules as the proxy CLI).
52
+ if (scenarioId && mode === "chaos") {
53
+ return reject("scenario_id and mode \"chaos\" cannot be combined: with a scenario set, chaos mode never injects anything.");
54
+ }
55
+ if (errorRate > 0 && mode !== "chaos") {
56
+ return reject("error_rate only has an effect together with mode \"chaos\".");
57
+ }
49
58
  if (scenarioId) {
50
59
  const scenario = getScenarioById(scenarioId);
51
60
  if (!scenario) {
@@ -60,7 +69,11 @@ export function handleStartSession(args) {
60
69
  };
61
70
  }
62
71
  }
63
- const session = createSession(scenarioId, mode, errorRate, args.write_tool_patterns);
72
+ const session = createSession(scenarioId, mode, errorRate, args.write_tool_patterns, {
73
+ readToolPatterns: args.read_tool_patterns,
74
+ keyFields: args.key_fields,
75
+ retryAfterSec: args.retry_after_sec,
76
+ });
64
77
  return {
65
78
  content: [
66
79
  {
@@ -89,11 +102,17 @@ export function handleExecuteCall(args) {
89
102
  isError: true,
90
103
  };
91
104
  }
105
+ const receivedAt = Date.now();
92
106
  const callNumber = incrementCall(args.session_id);
93
- if (args.idempotency_key && !session.scenarioId && session.mode !== "chaos") {
94
- const cached = findCommittedResponse(session, args.idempotency_key, args.tool_name, args.arguments);
107
+ // The key may arrive as the top-level idempotency_key parameter or inside
108
+ // `arguments`; both must behave the same.
109
+ const callKey = args.idempotency_key ?? extractKey(args.arguments, session.keyFields);
110
+ if (callKey && !session.scenarioId && session.mode !== "chaos") {
111
+ const cached = findCommittedResponse(session, callKey, args.tool_name, args.arguments);
95
112
  if (cached) {
96
- const dedupArgs = { ...args.arguments, idempotency_key: args.idempotency_key };
113
+ const dedupArgs = args.idempotency_key
114
+ ? { ...args.arguments, idempotency_key: args.idempotency_key }
115
+ : { ...args.arguments };
97
116
  const record = {
98
117
  callNumber,
99
118
  toolName: args.tool_name,
@@ -102,6 +121,7 @@ export function handleExecuteCall(args) {
102
121
  response: cached.response,
103
122
  upstreamExecuted: false,
104
123
  timestamp: Date.now(),
124
+ receivedAt,
105
125
  deduplicated: true,
106
126
  };
107
127
  addRecord(args.session_id, record);
@@ -138,13 +158,14 @@ export function handleExecuteCall(args) {
138
158
  let injectedError = null;
139
159
  let responseBody;
140
160
  let statusCode = 200;
141
- // Ghost-write: the simulated upstream actually executed the write, but the
142
- // agent only sees a timeout — it must read-before-retry to find out. Mirrors
143
- // the same classification used by the (unwired) proxy's decideChaosAction,
144
- // so F1 produces a real GHOST_CAUGHT/GHOST_MISSED instead of a generic retry.
161
+ // Ghost-write: the simulated backend actually executed the write, but the
162
+ // agent only sees a timeout. Mirrors proxy/ghost-write.ts::decideChaosAction.
145
163
  let isGhostWrite = false;
146
- const isWriteCall = isWriteTool(args.tool_name, session.writeToolPatterns);
147
- const priorWriteCalls = session.history.filter((r) => isWriteTool(r.toolName, session.writeToolPatterns)).length;
164
+ let scenarioTriggered = false;
165
+ let retryAfterSec;
166
+ const toolClass = classifyTool(session, args.tool_name);
167
+ const isWriteCall = toolClass === "write";
168
+ const priorWriteCalls = session.scenarioFireCount;
148
169
  if (session.scenarioId) {
149
170
  const scenario = getScenarioById(session.scenarioId);
150
171
  if (scenario &&
@@ -154,18 +175,20 @@ export function handleExecuteCall(args) {
154
175
  isWriteCall,
155
176
  priorWriteCalls,
156
177
  })) {
178
+ reserveScenarioFire(session);
157
179
  const resp = scenario.simulatedResponse;
180
+ scenarioTriggered = true;
158
181
  // Don't mark "success" type as injected error — F4 simulates successful duplicate dispatches
159
182
  injectedError = resp.type === "success" ? null : `[${resp.type.toUpperCase()}]`;
160
183
  statusCode = resp.statusCode ?? 500;
161
- responseBody = resp.body;
184
+ responseBody = resolveScenarioBody(scenario, session);
162
185
  isGhostWrite = resp.type === "timeout" && isWriteCall;
186
+ if (resp.type === "rate_limit")
187
+ retryAfterSec = session.retryAfterSec;
163
188
  // Note: resp.delayMs is NOT awaited on this synchronous path (unlike
164
189
  // proxy mode's interceptor.ts) — handleExecuteCall is called
165
- // synchronously from ~30 call sites across the test suite and
166
- // benchmark runner; making it async to honor a delay that no built-in
167
- // scenario currently sets to a nonzero value isn't worth that ripple.
168
- // Use `proxy` mode if simulated latency matters.
190
+ // synchronously from many call sites across the test suite and
191
+ // benchmark runner. Use `proxy` mode if simulated latency matters.
169
192
  }
170
193
  else {
171
194
  responseBody = {
@@ -180,8 +203,8 @@ export function handleExecuteCall(args) {
180
203
  injectedError = "[CHAOS_INJECTED]";
181
204
  statusCode = 500;
182
205
  responseBody = {
183
- error: "CHAOS_ERROR",
184
- message: "Stochastically injected failure.",
206
+ error: "INTERNAL_SERVER_ERROR",
207
+ message: "The server encountered an internal error.",
185
208
  };
186
209
  }
187
210
  else {
@@ -202,10 +225,19 @@ export function handleExecuteCall(args) {
202
225
  response,
203
226
  upstreamExecuted: isGhostWrite,
204
227
  timestamp: Date.now(),
228
+ receivedAt,
229
+ outcome: injectedError !== null ? "injected_failure" : "success",
230
+ toolClass,
231
+ // The simulated backend executes a write unless a failure was injected —
232
+ // except for a ghost-write, which executes although the agent sees one.
233
+ tookEffect: isWriteCall && (injectedError === null || isGhostWrite),
234
+ scenarioTriggered,
235
+ ...(retryAfterSec !== undefined ? { retryAfterSec } : {}),
205
236
  };
206
237
  addRecord(args.session_id, record);
207
- if (isGhostWrite && args.idempotency_key) {
208
- markGhostCommitted(args.session_id, args.idempotency_key);
238
+ const ghostKey = isGhostWrite ? extractKey(mergedArgs, session.keyFields) : undefined;
239
+ if (ghostKey) {
240
+ markGhostCommitted(args.session_id, ghostKey);
209
241
  }
210
242
  return {
211
243
  content: [
@@ -222,7 +254,12 @@ export function handleExecuteCall(args) {
222
254
  isError: statusCode >= 400,
223
255
  };
224
256
  }
225
- export async function handleGetReport(args) {
257
+ // `persist` writes the JSON report to .cuonztech/audit-report.json in the
258
+ // current working directory (the session-mode get_report tool's documented
259
+ // behavior). Proxy mode passes false: its report file is opt-in via
260
+ // --report-file, because a proxy started by an MCP client has no predictable
261
+ // working directory.
262
+ export async function handleGetReport(args, options = {}) {
226
263
  const session = getSession(args.session_id);
227
264
  if (!session) {
228
265
  return {
@@ -237,13 +274,14 @@ export async function handleGetReport(args) {
237
274
  }
238
275
  const report = generateReport(session);
239
276
  const format = args.format ?? "markdown";
240
- // Persist JSON report atomically
241
- try {
242
- await mkdir(dirname(AUDIT_REPORT_PATH), { recursive: true });
243
- await writeFile(AUDIT_REPORT_PATH, JSON.stringify(report, null, 2), "utf-8");
244
- }
245
- catch {
246
- // Non-fatal: report still returned in chat
277
+ if (options.persist ?? true) {
278
+ try {
279
+ await mkdir(dirname(AUDIT_REPORT_PATH), { recursive: true });
280
+ await writeFile(AUDIT_REPORT_PATH, JSON.stringify(report, null, 2), "utf-8");
281
+ }
282
+ catch {
283
+ // Non-fatal: report still returned in chat
284
+ }
247
285
  }
248
286
  const output = format === "json" ? JSON.stringify(report, null, 2) : formatMarkdown(report);
249
287
  return {
@@ -342,4 +380,3 @@ export function handleDeleteSession(args) {
342
380
  isError: !deleted,
343
381
  };
344
382
  }
345
- //# sourceMappingURL=handlers.js.map
@@ -20,14 +20,31 @@ export const StartSessionSchema = {
20
20
  .array(z.string())
21
21
  .optional()
22
22
  .describe('Glob patterns ("*" wildcard) deciding which tool names count as writes for ' +
23
- 'scenario triggering and idempotency classification, e.g. ["create_*", "send_*", "book_*"]. ' +
24
- 'Defaults to a broad set of common write-verb prefixes (write*, create_*, send_*, ' +
25
- 'submit_*, post_*, update_*, insert_*, book_*, pay_*, charge_*, cancel_*, delete_*, ' +
26
- 'confirm_*, place_*, add_*, register_*, schedule_*) — see DEFAULT_WRITE_TOOL_PATTERNS ' +
27
- 'in src/engine/state-machine.ts. Set this explicitly if your write tools use other naming.'),
23
+ 'scenario triggering and scoring, e.g. ["create_*", "send_*", "book_*"]. ' +
24
+ "Replaces the default list of common write-verb prefixes (write*, create_*, send_*, " +
25
+ "submit_*, post_*, update_*, book_*, pay_*, charge_*, transfer_*, refund_*, ...). " +
26
+ "A tool that matches " +
27
+ "neither a write pattern nor a read pattern is reported as unclassified and not scored."),
28
+ read_tool_patterns: z
29
+ .array(z.string())
30
+ .optional()
31
+ .describe('Extra glob patterns counted as reads, on top of the built-in read prefixes ' +
32
+ "(read, get, list, check, verify, query, fetch, inspect, status, search, find, " +
33
+ "lookup, describe, show, count, view, retrieve)."),
34
+ key_fields: z
35
+ .array(z.string())
36
+ .optional()
37
+ .describe('Argument names that carry an idempotency key (default ["idempotency_key"]; matched ' +
38
+ 'case-insensitively, ignoring "-" and "_"). They are ignored when two write calls ' +
39
+ "are compared, so the same write re-sent under a new key still counts as the same operation."),
40
+ retry_after_sec: z
41
+ .number()
42
+ .positive()
43
+ .optional()
44
+ .describe("retry_after (seconds) announced by scenario F2's 429 and enforced by the audit. Default 60."),
28
45
  };
29
46
  export const ExecuteCallSchema = {
30
- session_id: z.string().uuid().describe("Active session ID."),
47
+ session_id: z.string().uuid().describe("Active session ID (the UUID returned by start_session)."),
31
48
  tool_name: z.string().describe("Name of the tool the agent is calling."),
32
49
  arguments: z
33
50
  .record(z.string(), z.unknown())
@@ -38,7 +55,7 @@ export const ExecuteCallSchema = {
38
55
  .describe("Idempotency key for duplicate detection."),
39
56
  };
40
57
  export const GetReportSchema = {
41
- session_id: z.string().uuid().describe("Session ID to audit."),
58
+ session_id: z.string().uuid().describe("Session ID to audit (the UUID returned by start_session)."),
42
59
  format: z
43
60
  .enum(["json", "markdown"])
44
61
  .optional()
@@ -46,7 +63,7 @@ export const GetReportSchema = {
46
63
  .describe("Output format: markdown (default) or json."),
47
64
  };
48
65
  export const GetScoreSchema = {
49
- session_id: z.string().uuid().describe("Session ID to score."),
66
+ session_id: z.string().uuid().describe("Session ID to score (the UUID returned by start_session)."),
50
67
  format: z
51
68
  .enum(["json", "markdown"])
52
69
  .optional()
@@ -55,9 +72,8 @@ export const GetScoreSchema = {
55
72
  };
56
73
  export const ListScenariosSchema = {};
57
74
  export const ResetSessionSchema = {
58
- session_id: z.string().uuid().describe("Session ID to reset."),
75
+ session_id: z.string().uuid().describe("Session ID to reset (the UUID returned by start_session)."),
59
76
  };
60
77
  export const DeleteSessionSchema = {
61
- session_id: z.string().uuid().describe("Session ID to delete."),
78
+ session_id: z.string().uuid().describe("Session ID to delete (the UUID returned by start_session)."),
62
79
  };
63
- //# sourceMappingURL=schemas.js.map
package/dist/version.js CHANGED
@@ -10,4 +10,3 @@ import { dirname, join } from "node:path";
10
10
  const __dirname = dirname(fileURLToPath(import.meta.url));
11
11
  const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8"));
12
12
  export const VERSION = pkg.version;
13
- //# sourceMappingURL=version.js.map
package/package.json CHANGED
@@ -1,10 +1,11 @@
1
1
  {
2
2
  "name": "cuonztech-agent-harness",
3
- "version": "0.4.0",
4
- "description": "MCP-Server zur Härtung autonomer KI-Agenten — simulierte F1-F5-Chaos-Szenarien, Idempotenz-Prüfung, Ghost-Write-Erkennung, striktes Key-genaues State-Diffing, Resilience Score, automatische System-Prompt-Reparatur und ein transparenter Chaos-Proxy vor einem echten Upstream-MCP-Server.",
3
+ "version": "0.5.1",
4
+ "description": "Chaos-Proxy für MCP-Server: injiziert F1-F5-Fehlerszenarien (Ghost-Write nach Timeout, 429, defektes JSON, Duplicate Dispatch, 500) vor einem echten Upstream-MCP-Server und prüft, ob ein Agent Schreibvorgänge doppelt ausführt. Mit Resilience Score und System-Prompt-Patches für beobachtetes Fehlverhalten.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
7
7
  "bin": {
8
+ "cuonztech-agent-harness": "dist/index.js",
8
9
  "cuonztech-harness": "dist/index.js"
9
10
  },
10
11
  "scripts": {
@@ -42,7 +43,7 @@
42
43
  "node": ">=20"
43
44
  },
44
45
  "files": [
45
- "dist",
46
+ "dist/**/*.js",
46
47
  "README.md",
47
48
  "LICENSE"
48
49
  ],
@@ -1,13 +0,0 @@
1
- import type { ResilienceScore } from "./score.js";
2
- import type { HardeningPatch } from "../hardening/patch-generator.js";
3
- export declare function formatCTA(score: ResilienceScore): string;
4
- export interface CuonzTechTelemetry {
5
- cuonztechScore: number;
6
- verdict: "UNVERIFIED_FOR_PRODUCTION";
7
- auditProvider: string;
8
- recommendedAction: string;
9
- isReferenceRun: true;
10
- note: string;
11
- }
12
- export declare function buildTelemetry(score: ResilienceScore, patches: HardeningPatch[]): CuonzTechTelemetry;
13
- //# sourceMappingURL=cta.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"cta.d.ts","sourceRoot":"","sources":["../../src/benchmark/cta.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAyBtE,wBAAgB,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAsBxD;AAED,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,2BAA2B,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAM1B,cAAc,EAAE,IAAI,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,cAAc,CAC5B,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,cAAc,EAAE,GACxB,kBAAkB,CAoBpB"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"cta.js","sourceRoot":"","sources":["../../src/benchmark/cta.ts"],"names":[],"mappings":"AAKA,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAC5B,MAAM,SAAS,GAAG,4BAA4B,CAAC;AAC/C,MAAM,cAAc,GAAG,8CAA8C,CAAC;AAEtE,SAAS,aAAa,CAAC,OAAe;IACpC,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,WAAW,CAAC;IACtC,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAgC;IAC3D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,EAAE,CAAC,WAAW,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,wBAAwB,CAAC,CAAC;IACjF,IAAI,EAAE,CAAC,YAAY,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY,iBAAiB,CAAC,CAAC;IAC5E,IAAI,EAAE,CAAC,cAAc,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,cAAc,qBAAqB,CAAC,CAAC;IACpF,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC;AAC9E,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAgC;IAC3D,OAAO,EAAE,CAAC,WAAW,GAAG,CAAC,IAAI,EAAE,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,CAAC,cAAc,GAAG,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAsB;IAC9C,MAAM,KAAK,GAAG;QACZ,GAAG;QACH,+BAA+B,KAAK,CAAC,OAAO,SAAS,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG;QACpF,kBAAkB,KAAK,CAAC,eAAe,KAAK,mBAAmB,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE;QAClF,kBAAkB,KAAK,CAAC,YAAY,GAAG;QACvC,kBAAkB,KAAK,CAAC,YAAY,GAAG;QACvC,IAAI;KACL,CAAC;IAEF,IAAI,mBAAmB,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,GAAG,EAAE,EAAE,CAAC;QAC/D,KAAK,CAAC,IAAI,CACR,6DAA6D,EAC7D,qDAAqD,EACrD,OAAO,SAAS,EAAE,CACnB,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAgBD,MAAM,UAAU,cAAc,CAC5B,KAAsB,EACtB,OAAyB;IAEzB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,iBAAiB,GAAG,QAAQ;QAChC,CAAC,CAAC,0BAA0B,QAAQ,CAAC,QAAQ,iDAAiD,SAAS,IAAI;QAC3G,CAAC,CAAC,4HAA4H,SAAS,IAAI,CAAC;IAE9I,OAAO;QACL,cAAc,EAAE,KAAK,CAAC,OAAO;QAC7B,wEAAwE;QACxE,2EAA2E;QAC3E,uEAAuE;QACvE,OAAO,EAAE,2BAA2B;QACpC,aAAa,EAAE,cAAc;QAC7B,iBAAiB;QACjB,cAAc,EAAE,IAAI;QACpB,IAAI,EACF,2EAA2E;YAC3E,mEAAmE;YACnE,qCAAqC;KACxC,CAAC;AACJ,CAAC"}
@@ -1,19 +0,0 @@
1
- import { type ResilienceScore } from "./score.js";
2
- import { type HardeningPatch } from "../hardening/patch-generator.js";
3
- import type { AuditReport } from "../core/report-generator.js";
4
- export interface BenchmarkConfig {
5
- scenarios: string[];
6
- runsPerScenario: number;
7
- jitterMs: number;
8
- }
9
- export interface BenchmarkResult {
10
- reports: AuditReport[];
11
- score: ResilienceScore;
12
- patches: HardeningPatch[];
13
- markdownReport: string;
14
- markdownScore: string;
15
- markdownPatches: string;
16
- }
17
- export declare function defaultConfig(): BenchmarkConfig;
18
- export declare function runBenchmark(config?: BenchmarkConfig): Promise<BenchmarkResult>;
19
- //# sourceMappingURL=runner.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAMA,OAAO,EAAmC,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AACnF,OAAO,EAAsC,KAAK,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAC1G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,KAAK,EAAE,eAAe,CAAC;IACvB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,aAAa,IAAI,eAAe,CAM/C;AAWD,wBAAsB,YAAY,CAChC,MAAM,GAAE,eAAiC,GACxC,OAAO,CAAC,eAAe,CAAC,CA0C1B"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAwB,MAAM,YAAY,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAuB,MAAM,iCAAiC,CAAC;AAkB1G,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;QACzC,eAAe,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAC;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,MAA4C;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO;IACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAM,GAAoB,aAAa,EAAE;IAEzC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QAC1C,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,eAAe,EAAE,GAAG,EAAE,EAAE,CAAC;YACtD,6BAA6B;YAC7B,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE;gBAAE,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAE3D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAChC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;gBAC7C,CAAC,CAAC,CAAC,CAAC;YAEN,gBAAgB;YAChB,MAAM,IAAI,GAAG,SAAS,CACpB,kBAAkB,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CACvB,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC;YAE3B,yCAAyC;YACzC,MAAM,aAAa,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAE7C,aAAa;YACb,MAAM,MAAM,GAAG,SAAS,CACtB,MAAM,eAAe,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAC5C,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAEhD,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,cAAc,EAAE,OAAO;aACpB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,UAAU,IAAI,QAAQ,OAAO,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;aACzF,IAAI,CAAC,aAAa,CAAC;QACtB,aAAa,EAAE,iBAAiB,CAAC,KAAK,CAAC;QACvC,eAAe,EAAE,iBAAiB,CAAC,OAAO,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,SAAiB,EACjB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,GAAG,GAAG,SAAS,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAEhD,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,IAAI,EAAE,CAAC;YACV,mDAAmD;YACnD,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;YACvD,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,sEAAsE;YACtE,oEAAoE;YACpE,mEAAmE;YACnE,mEAAmE;YACnE,+DAA+D;YAC/D,oDAAoD;YACpD,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,oBAAoB;YAC3E,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,4BAA4B;YACnF,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,6BAA6B;YAC7B,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,gDAAgD;YAChD,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;YACxD,MAAM;QACR,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,gCAAgC;YAChC,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,MAAM;QACR,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CACX,SAAiB,EACjB,QAAgB,EAChB,IAA6B,EAC7B,cAAuB;IAEvB,iBAAiB,CAAC;QAChB,UAAU,EAAE,SAAS;QACrB,SAAS,EAAE,QAAQ;QACnB,SAAS,EAAE,IAAI;QACf,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAc;IACxC,IAAI,EAAE,GAAG,iCAAiC,CAAC;IAC3C,EAAE,IAAI,eAAe,CAAC,CAAC,OAAO,MAAM,CAAC;IACrC,EAAE,IAAI,mBAAmB,CAAC,CAAC,UAAU,MAAM,CAAC;IAC5C,EAAE,IAAI,uBAAuB,CAAC,CAAC,cAAc,MAAM,CAAC;IACpD,EAAE,IAAI,qBAAqB,CAAC,CAAC,YAAY,MAAM,CAAC;IAChD,EAAE,IAAI,qBAAqB,CAAC,CAAC,YAAY,MAAM,CAAC;IAChD,EAAE,IAAI,uBAAuB,CAAC,CAAC,cAAc,MAAM,CAAC;IACpD,EAAE,IAAI,oBAAoB,CAAC,CAAC,WAAW,MAAM,CAAC;IAC9C,OAAO,EAAE,CAAC;AACZ,CAAC"}
@@ -1,26 +0,0 @@
1
- import type { AuditReport } from "../core/report-generator.js";
2
- export interface ResilienceScore {
3
- overall: number;
4
- idempotencyRate: number;
5
- recoveryRate: number;
6
- honestyScore: number;
7
- breakdown: ScoreBreakdown;
8
- }
9
- export interface ScoreBreakdown {
10
- totalScenarios: number;
11
- passedScenarios: number;
12
- failedScenarios: number;
13
- validRetries: number;
14
- blindRetries: number;
15
- redundantCalls: number;
16
- ghostWrites: number;
17
- ghostCaught: number;
18
- ghostMissed: number;
19
- injectedErrors: number;
20
- recoveredFromErrors: number;
21
- hallucinatedVerifications: number;
22
- syntaxCrashes: number;
23
- }
24
- export declare function computeScore(reports: AuditReport[]): ResilienceScore;
25
- export declare function formatScoreReport(score: ResilienceScore): string;
26
- //# sourceMappingURL=score.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/benchmark/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,yBAAyB,EAAE,MAAM,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,eAAe,CAgDpE;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAyBhE"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/benchmark/score.ts"],"names":[],"mappings":"AA0BA,MAAM,UAAU,YAAY,CAAC,OAAsB;IACjD,MAAM,EAAE,GAAmB;QACzB,cAAc,EAAE,OAAO,CAAC,MAAM;QAC9B,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;QACnE,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;QACnE,YAAY,EAAE,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC;QAC1C,YAAY,EAAE,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC;QAC1C,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC;QAC9C,WAAW,EAAE,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC;QACxC,WAAW,EAAE,GAAG,CAAC,OAAO,EAAE,sBAAsB,CAAC;QACjD,WAAW,EAAE,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC;QAC7C,cAAc,EAAE,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC;QAC9C,mBAAmB,EAAE,GAAG,CAAC,OAAO,EAAE,qBAAqB,CAAC;QACxD,yBAAyB,EAAE,GAAG,CAAC,OAAO,EAAE,2BAA2B,CAAC;QACpE,aAAa,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC;KAC7C,CAAC;IAEF,8EAA8E;IAC9E,MAAM,aAAa,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,cAAc,GAAG,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,CAAC;IAC9G,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,CAAC;IACtD,MAAM,eAAe,GAAG,aAAa,KAAK,CAAC;QACzC,CAAC,CAAC,GAAG,CAAC,yBAAyB;QAC/B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC;IAErD,uEAAuE;IACvE,MAAM,YAAY,GAAG,EAAE,CAAC,cAAc,KAAK,CAAC;QAC1C,CAAC,CAAC,GAAG,CAAC,+BAA+B;QACrC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,mBAAmB,GAAG,EAAE,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;IAEnE,sDAAsD;IACtD,MAAM,kBAAkB,GAAG,EAAE,CAAC,cAAc,CAAC;IAC7C,MAAM,iBAAiB,GAAG,EAAE,CAAC,yBAAyB,GAAG,EAAE,CAAC,aAAa,CAAC;IAC1E,MAAM,YAAY,GAAG,kBAAkB,KAAK,CAAC;QAC3C,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,GAAG,iBAAiB,CAAC,GAAG,kBAAkB,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAEnG,4BAA4B;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,eAAe,GAAG,GAAG,GAAG,YAAY,GAAG,IAAI,GAAG,YAAY,GAAG,IAAI,CAClE,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC;QAC/C,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,GAAG,CAAC;QACzC,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,GAAG,CAAC;QACzC,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAsB;IACtD,IAAI,EAAE,GAAG,kCAAkC,CAAC;IAC5C,EAAE,IAAI,eAAe,KAAK,CAAC,OAAO,UAAU,CAAC;IAC7C,EAAE,IAAI,iDAAiD,CAAC;IACxD,EAAE,IAAI,wBAAwB,KAAK,CAAC,eAAe,aAAa,CAAC;IACjE,EAAE,IAAI,qBAAqB,KAAK,CAAC,YAAY,aAAa,CAAC;IAC3D,EAAE,IAAI,qBAAqB,KAAK,CAAC,YAAY,aAAa,CAAC;IAE3D,EAAE,IAAI,oBAAoB,CAAC;IAC3B,EAAE,IAAI,iCAAiC,CAAC;IACxC,EAAE,IAAI,qBAAqB,KAAK,CAAC,SAAS,CAAC,cAAc,MAAM,CAAC;IAChE,EAAE,IAAI,cAAc,KAAK,CAAC,SAAS,CAAC,eAAe,MAAM,CAAC;IAC1D,EAAE,IAAI,cAAc,KAAK,CAAC,SAAS,CAAC,eAAe,MAAM,CAAC;IAC1D,EAAE,IAAI,qBAAqB,KAAK,CAAC,SAAS,CAAC,YAAY,MAAM,CAAC;IAC9D,EAAE,IAAI,qBAAqB,KAAK,CAAC,SAAS,CAAC,YAAY,MAAM,CAAC;IAC9D,EAAE,IAAI,uBAAuB,KAAK,CAAC,SAAS,CAAC,cAAc,MAAM,CAAC;IAClE,EAAE,IAAI,oBAAoB,KAAK,CAAC,SAAS,CAAC,WAAW,MAAM,CAAC;IAC5D,EAAE,IAAI,oBAAoB,KAAK,CAAC,SAAS,CAAC,WAAW,MAAM,CAAC;IAC5D,EAAE,IAAI,oBAAoB,KAAK,CAAC,SAAS,CAAC,WAAW,MAAM,CAAC;IAC5D,EAAE,IAAI,uBAAuB,KAAK,CAAC,SAAS,CAAC,cAAc,MAAM,CAAC;IAClE,EAAE,IAAI,iBAAiB,KAAK,CAAC,SAAS,CAAC,mBAAmB,MAAM,CAAC;IACjE,EAAE,IAAI,sBAAsB,KAAK,CAAC,SAAS,CAAC,yBAAyB,MAAM,CAAC;IAC5E,EAAE,IAAI,sBAAsB,KAAK,CAAC,SAAS,CAAC,aAAa,MAAM,CAAC;IAEhE,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,GAAG,CAAC,OAAsB,EAAE,GAAsB;IACzD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,CAAC,CAAC,GAAG,CAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,KAAK,CAAC,GAAW,EAAE,GAAW,EAAE,GAAW;IAClD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC3C,CAAC"}
@@ -1,47 +0,0 @@
1
- import { type SessionState } from "../engine/state-machine.js";
2
- export type DuplicateClass = "VALID_RETRY" | "BLIND_RETRY" | "REDUNDANT_CALL" | "GHOST_WRITE" | "GHOST_CAUGHT" | "GHOST_MISSED";
3
- export interface AuditReport {
4
- sessionId: string;
5
- scenarioId: string | null;
6
- mode: string;
7
- totalCalls: number;
8
- upstreamCalls: number;
9
- validRetries: number;
10
- blindRetries: number;
11
- redundantCalls: number;
12
- ghostWrites: number;
13
- ghostWriteDetections: number;
14
- ghostWriteMisses: number;
15
- hallucinatedVerifications: number;
16
- syntaxCrashes: number;
17
- injectedErrors: number;
18
- recoveredFromErrors: number;
19
- verdict: "PASS" | "FAIL";
20
- violations: Violation[];
21
- keySummary: KeySummaryEntry[];
22
- generatedAt: string;
23
- }
24
- export interface Violation {
25
- callNumber: number;
26
- type: DuplicateClass | "HALLUCINATED_VERIFICATION" | "SYNTAX_CRASH";
27
- description: string;
28
- idempotencyKey?: string;
29
- }
30
- export interface KeySummaryEntry {
31
- key: string;
32
- state: string;
33
- writeCalls: number;
34
- lastError: string | null;
35
- readBeforeRetry: boolean;
36
- ghostCommitted: boolean;
37
- classification: DuplicateClass | "INITIAL" | "WRITE_ONLY";
38
- stateDiff: {
39
- recordExisted: boolean;
40
- wasOverwritten: boolean;
41
- didAgentReadBefore: boolean;
42
- agentGaveUpSilently: boolean;
43
- };
44
- }
45
- export declare function generateReport(session: SessionState): AuditReport;
46
- export declare function formatMarkdown(report: AuditReport): string;
47
- //# sourceMappingURL=report-generator.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"report-generator.d.ts","sourceRoot":"","sources":["../../src/core/report-generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAkD,MAAM,4BAA4B,CAAC;AAE/G,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,aAAa,GACb,gBAAgB,GAChB,aAAa,GACb,cAAc,GACd,cAAc,CAAC;AAEnB,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IAEtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IAEzB,yBAAyB,EAAE,MAAM,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,cAAc,GAAG,2BAA2B,GAAG,cAAc,CAAC;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,EAAE,cAAc,GAAG,SAAS,GAAG,YAAY,CAAC;IAQ1D,SAAS,EAAE;QACT,aAAa,EAAE,OAAO,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC;QACxB,kBAAkB,EAAE,OAAO,CAAC;QAC5B,mBAAmB,EAAE,OAAO,CAAC;KAC9B,CAAC;CACH;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,WAAW,CAkLjE;AAqJD,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CA+C1D"}