cuonztech-agent-harness 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -7
- package/dist/benchmark/cta.d.ts +2 -0
- package/dist/benchmark/cta.d.ts.map +1 -1
- package/dist/benchmark/cta.js +4 -0
- package/dist/benchmark/cta.js.map +1 -1
- package/dist/benchmark/runner.js +9 -4
- package/dist/benchmark/runner.js.map +1 -1
- package/dist/core/report-generator.d.ts.map +1 -1
- package/dist/core/report-generator.js +42 -17
- package/dist/core/report-generator.js.map +1 -1
- package/dist/engine/state-machine.d.ts +5 -2
- package/dist/engine/state-machine.d.ts.map +1 -1
- package/dist/engine/state-machine.js +50 -4
- package/dist/engine/state-machine.js.map +1 -1
- package/dist/index.js +134 -5
- package/dist/index.js.map +1 -1
- package/dist/proxy/gateway.d.ts.map +1 -1
- package/dist/proxy/gateway.js +2 -1
- package/dist/proxy/gateway.js.map +1 -1
- package/dist/proxy/ghost-write.d.ts.map +1 -1
- package/dist/proxy/ghost-write.js +15 -4
- package/dist/proxy/ghost-write.js.map +1 -1
- package/dist/proxy/interceptor.d.ts +3 -4
- package/dist/proxy/interceptor.d.ts.map +1 -1
- package/dist/proxy/interceptor.js +71 -42
- package/dist/proxy/interceptor.js.map +1 -1
- package/dist/scenarios/f1-f5.d.ts.map +1 -1
- package/dist/scenarios/f1-f5.js +18 -6
- package/dist/scenarios/f1-f5.js.map +1 -1
- package/dist/scenarios/types.d.ts +7 -1
- package/dist/scenarios/types.d.ts.map +1 -1
- package/dist/server/server.d.ts.map +1 -1
- package/dist/server/server.js +2 -1
- package/dist/server/server.js.map +1 -1
- package/dist/tools/handlers.d.ts +1 -0
- package/dist/tools/handlers.d.ts.map +1 -1
- package/dist/tools/handlers.js +17 -3
- package/dist/tools/handlers.js.map +1 -1
- package/dist/tools/schemas.d.ts +1 -0
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +9 -0
- package/dist/tools/schemas.js.map +1 -1
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +13 -0
- package/dist/version.js.map +1 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5,11 +5,42 @@ import { runBenchmark, defaultConfig } from "./benchmark/runner.js";
|
|
|
5
5
|
import { formatCTA, buildTelemetry } from "./benchmark/cta.js";
|
|
6
6
|
import { ProxyInterceptor } from "./proxy/interceptor.js";
|
|
7
7
|
import { createProxyServer } from "./proxy/gateway.js";
|
|
8
|
+
import { getScenarioById, ALL_SCENARIOS } from "./scenarios/f1-f5.js";
|
|
9
|
+
import { DEFAULT_WRITE_TOOL_PATTERNS, matchesAnyGlob } from "./engine/state-machine.js";
|
|
8
10
|
import { writeFile, mkdir } from "node:fs/promises";
|
|
9
11
|
const args = process.argv.slice(2);
|
|
12
|
+
function printHelp() {
|
|
13
|
+
process.stdout.write(`cuonztech-agent-harness — chaos/idempotency test harness for MCP agents\n\n` +
|
|
14
|
+
`Usage:\n` +
|
|
15
|
+
` cuonztech-agent-harness Start as an MCP server (stdio)\n` +
|
|
16
|
+
` cuonztech-agent-harness benchmark [opts] Run the fixed reference self-test\n` +
|
|
17
|
+
` cuonztech-agent-harness proxy --upstream-command <cmd> [opts]\n` +
|
|
18
|
+
` Run as a transparent chaos proxy\n` +
|
|
19
|
+
` in front of a real upstream MCP server\n\n` +
|
|
20
|
+
`benchmark options:\n` +
|
|
21
|
+
` --scenarios F1,F2,... Which scenarios to run (default: all)\n` +
|
|
22
|
+
` --runs <n> Runs per scenario, positive integer (default: 1)\n` +
|
|
23
|
+
` --jitter <ms> Random delay between scripted calls (default: 0)\n\n` +
|
|
24
|
+
`proxy options:\n` +
|
|
25
|
+
` --upstream-command <cmd> Command to spawn the upstream MCP server (required)\n` +
|
|
26
|
+
` --upstream-args <a,b,c> Comma-separated args for the upstream command\n` +
|
|
27
|
+
` --upstream-cwd <dir> Working directory for the upstream process\n` +
|
|
28
|
+
` --upstream-env KEY=VAL,... Extra/override env vars for the upstream process\n` +
|
|
29
|
+
` --scenario F1..F5 Activate one deterministic scenario\n` +
|
|
30
|
+
` --mode chaos Stochastic error injection instead of a scenario\n` +
|
|
31
|
+
` --error-rate <0..1> Error probability in chaos mode (default: 0)\n` +
|
|
32
|
+
` --write-tools pat1,pat2,... Glob patterns ("*") for which tool names count as\n` +
|
|
33
|
+
` writes (default covers common verbs: write*,\n` +
|
|
34
|
+
` create_*, send_*, submit_*, post_*, update_*, ...\n` +
|
|
35
|
+
` — see DEFAULT_WRITE_TOOL_PATTERNS). Set this if\n` +
|
|
36
|
+
` your upstream's write tools use other naming.\n`);
|
|
37
|
+
}
|
|
10
38
|
async function main() {
|
|
11
39
|
const mode = args[0];
|
|
12
|
-
if (mode === "
|
|
40
|
+
if (mode === "--help" || mode === "-h" || mode === "help") {
|
|
41
|
+
printHelp();
|
|
42
|
+
}
|
|
43
|
+
else if (mode === "benchmark" || mode === "evaluate") {
|
|
13
44
|
await runBenchmarkMode();
|
|
14
45
|
}
|
|
15
46
|
else if (mode === "proxy") {
|
|
@@ -34,21 +65,82 @@ async function runProxyMode() {
|
|
|
34
65
|
const upstreamArgs = argsIdx !== -1 && args[argsIdx + 1] ? args[argsIdx + 1].split(",") : [];
|
|
35
66
|
const cwdIdx = args.indexOf("--upstream-cwd");
|
|
36
67
|
const upstreamCwd = cwdIdx !== -1 ? args[cwdIdx + 1] : undefined;
|
|
68
|
+
const envIdx = args.indexOf("--upstream-env");
|
|
69
|
+
const upstreamEnv = {};
|
|
70
|
+
if (envIdx !== -1 && args[envIdx + 1]) {
|
|
71
|
+
for (const pair of args[envIdx + 1].split(",")) {
|
|
72
|
+
const eq = pair.indexOf("=");
|
|
73
|
+
if (eq === -1) {
|
|
74
|
+
process.stderr.write(`Error: --upstream-env entry "${pair}" is not KEY=VALUE.\n`);
|
|
75
|
+
process.exit(1);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
upstreamEnv[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const writeToolsIdx = args.indexOf("--write-tools");
|
|
82
|
+
const writeToolPatterns = writeToolsIdx !== -1 && args[writeToolsIdx + 1]
|
|
83
|
+
? args[writeToolsIdx + 1].split(",")
|
|
84
|
+
: undefined;
|
|
37
85
|
const scenarioIdx = args.indexOf("--scenario");
|
|
38
86
|
const scenarioId = scenarioIdx !== -1 ? args[scenarioIdx + 1] : undefined;
|
|
87
|
+
if (scenarioId && !getScenarioById(scenarioId)) {
|
|
88
|
+
process.stderr.write(`Error: unknown scenario "${scenarioId}". Valid: ${ALL_SCENARIOS.map((s) => s.id).join(", ")}\n`);
|
|
89
|
+
process.exit(1);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
39
92
|
const modeIdx = args.indexOf("--mode");
|
|
40
|
-
const
|
|
93
|
+
const chaosModeRequested = modeIdx !== -1 ? args[modeIdx + 1] : undefined;
|
|
94
|
+
const chaosMode = chaosModeRequested === "chaos" ? "chaos" : "deterministic";
|
|
41
95
|
const errorRateIdx = args.indexOf("--error-rate");
|
|
42
96
|
const errorRate = errorRateIdx !== -1 && args[errorRateIdx + 1]
|
|
43
97
|
? parseFloat(args[errorRateIdx + 1])
|
|
44
98
|
: 0.0;
|
|
99
|
+
if (chaosMode === "chaos" && errorRateIdx === -1) {
|
|
100
|
+
process.stderr.write("[cuonztech-agent-harness] WARNING: --mode chaos without --error-rate defaults " +
|
|
101
|
+
"to a 0% error rate — this run will inject nothing. Pass e.g. --error-rate 0.2 " +
|
|
102
|
+
"to actually trigger stochastic failures.\n");
|
|
103
|
+
}
|
|
104
|
+
if (scenarioId || chaosMode === "chaos") {
|
|
105
|
+
process.stderr.write("[cuonztech-agent-harness] WARNING: chaos/scenario injection is active — " +
|
|
106
|
+
"the agent will sometimes receive errors or duplicate successes that do " +
|
|
107
|
+
"NOT reflect what actually happened upstream. Only point --upstream-command " +
|
|
108
|
+
"at a disposable test/staging backend, never production.\n");
|
|
109
|
+
}
|
|
45
110
|
const interceptor = new ProxyInterceptor({
|
|
46
|
-
upstream: {
|
|
111
|
+
upstream: {
|
|
112
|
+
command: upstreamCommand,
|
|
113
|
+
args: upstreamArgs,
|
|
114
|
+
cwd: upstreamCwd,
|
|
115
|
+
env: upstreamEnv,
|
|
116
|
+
},
|
|
47
117
|
scenarioId,
|
|
48
118
|
mode: chaosMode,
|
|
49
119
|
errorRate,
|
|
120
|
+
writeToolPatterns,
|
|
50
121
|
});
|
|
51
122
|
await interceptor.connectUpstream();
|
|
123
|
+
// If NOT ONE of the upstream's real tools matches any configured
|
|
124
|
+
// write-tool pattern, ghost-write/duplicate-dispatch detection (F1/F4) and
|
|
125
|
+
// idempotency-key tracking will NEVER trigger for this upstream — the
|
|
126
|
+
// session stays silently inert while get_report/get_score can still print
|
|
127
|
+
// a misleadingly clean 100/100. An adversarial test against the published
|
|
128
|
+
// package found exactly this against a realistically-named tool. Warn
|
|
129
|
+
// loudly instead of staying silent, even though DEFAULT_WRITE_TOOL_PATTERNS
|
|
130
|
+
// is now broader than just "write*".
|
|
131
|
+
const effectivePatterns = writeToolPatterns && writeToolPatterns.length > 0
|
|
132
|
+
? writeToolPatterns
|
|
133
|
+
: DEFAULT_WRITE_TOOL_PATTERNS;
|
|
134
|
+
const upstreamTools = interceptor.getUpstreamTools();
|
|
135
|
+
const anyToolMatches = upstreamTools.some((t) => matchesAnyGlob(t.name, effectivePatterns));
|
|
136
|
+
if (upstreamTools.length > 0 && !anyToolMatches) {
|
|
137
|
+
process.stderr.write(`[cuonztech-agent-harness] WARNING: none of the upstream's ${upstreamTools.length} ` +
|
|
138
|
+
`tool(s) (${upstreamTools.map((t) => t.name).join(", ")}) match any write-tool ` +
|
|
139
|
+
`pattern (currently: ${effectivePatterns.join(", ")}). Ghost-write/duplicate-dispatch ` +
|
|
140
|
+
`detection will NEVER trigger for this upstream, and real duplicate writes will go ` +
|
|
141
|
+
`completely undetected. If your write tools use different naming, pass e.g. ` +
|
|
142
|
+
`--write-tools "yourprefix_*".\n`);
|
|
143
|
+
}
|
|
52
144
|
const shutdown = async () => {
|
|
53
145
|
await interceptor.disconnectUpstream();
|
|
54
146
|
process.exit(0);
|
|
@@ -60,20 +152,57 @@ async function runProxyMode() {
|
|
|
60
152
|
}
|
|
61
153
|
async function runBenchmarkMode() {
|
|
62
154
|
const config = defaultConfig();
|
|
155
|
+
// Strict integer parsing: parseInt() alone is too lenient (parseInt("1.5")
|
|
156
|
+
// silently truncates to 1 instead of being rejected) — only a string
|
|
157
|
+
// matching /^-?\d+$/ counts as a valid integer here.
|
|
158
|
+
function parseStrictInt(raw) {
|
|
159
|
+
return /^-?\d+$/.test(raw) ? parseInt(raw, 10) : null;
|
|
160
|
+
}
|
|
63
161
|
// Parse optional flags
|
|
64
162
|
const runsIdx = args.indexOf("--runs");
|
|
65
163
|
if (runsIdx !== -1 && args[runsIdx + 1]) {
|
|
66
|
-
|
|
164
|
+
const parsed = parseStrictInt(args[runsIdx + 1]);
|
|
165
|
+
if (parsed === null || parsed < 1) {
|
|
166
|
+
// A non-positive/NaN/non-integer value used to silently run ZERO
|
|
167
|
+
// scenarios while still printing a fake "Overall: 100/100 [EXCELLENT]"
|
|
168
|
+
// (computeScore's "no data = perfect" default, which is correct for a
|
|
169
|
+
// real session with genuinely nothing to flag, but dishonest here —
|
|
170
|
+
// nothing was ever tested) and persisting that 100/100 into
|
|
171
|
+
// .cuonztech/benchmark-report.json for anything (e.g. a CI gate) to
|
|
172
|
+
// pick up as real.
|
|
173
|
+
process.stderr.write(`Error: --runs must be a positive integer, got "${args[runsIdx + 1]}".\n`);
|
|
174
|
+
process.exit(1);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
config.runsPerScenario = parsed;
|
|
67
178
|
}
|
|
68
179
|
const jitterIdx = args.indexOf("--jitter");
|
|
69
180
|
if (jitterIdx !== -1 && args[jitterIdx + 1]) {
|
|
70
|
-
|
|
181
|
+
const parsed = parseStrictInt(args[jitterIdx + 1]);
|
|
182
|
+
if (parsed === null || parsed < 0) {
|
|
183
|
+
process.stderr.write(`Error: --jitter must be a non-negative integer (ms), got "${args[jitterIdx + 1]}".\n`);
|
|
184
|
+
process.exit(1);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
config.jitterMs = parsed;
|
|
71
188
|
}
|
|
72
189
|
const scenariosIdx = args.indexOf("--scenarios");
|
|
73
190
|
if (scenariosIdx !== -1 && args[scenariosIdx + 1]) {
|
|
74
191
|
config.scenarios = args[scenariosIdx + 1].split(",");
|
|
75
192
|
}
|
|
193
|
+
const unknown = config.scenarios.filter((id) => !getScenarioById(id));
|
|
194
|
+
if (unknown.length > 0) {
|
|
195
|
+
process.stderr.write(`Error: unknown scenario(s) "${unknown.join(", ")}". ` +
|
|
196
|
+
`Valid: ${ALL_SCENARIOS.map((s) => s.id).join(", ")}\n`);
|
|
197
|
+
process.exit(1);
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
76
200
|
process.stdout.write(`CuonzTech Agent-Harness Benchmark\n`);
|
|
201
|
+
process.stdout.write("NOTE: this replays a fixed built-in reference call sequence — it does NOT\n" +
|
|
202
|
+
"connect to or measure your own agent. It is a self-test of the harness's\n" +
|
|
203
|
+
"scenario/scoring logic, useful as a reference point, not a verdict on your\n" +
|
|
204
|
+
"system. To score YOUR agent, connect it to `proxy` mode or the MCP server's\n" +
|
|
205
|
+
"`start_session`/`execute_call`/`get_score` tools instead.\n\n");
|
|
77
206
|
process.stdout.write(`Scenarios: ${config.scenarios.join(", ")}\n`);
|
|
78
207
|
process.stdout.write(`Runs per scenario: ${config.runsPerScenario}\n`);
|
|
79
208
|
process.stdout.write(`Jitter: ${config.jitterMs}ms\n\n`);
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAErB,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AACxF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,SAAS,SAAS;IAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6EAA6E;QAC3E,UAAU;QACV,8EAA8E;QAC9E,kFAAkF;QAClF,mEAAmE;QACnE,iFAAiF;QACjF,yFAAyF;QACzF,sBAAsB;QACtB,mEAAmE;QACnE,8EAA8E;QAC9E,gFAAgF;QAChF,kBAAkB;QAClB,wFAAwF;QACxF,kFAAkF;QAClF,+EAA+E;QAC/E,qFAAqF;QACrF,wEAAwE;QACxE,qFAAqF;QACrF,kFAAkF;QAClF,sFAAsF;QACtF,kFAAkF;QAClF,uFAAuF;QACvF,qFAAqF;QACrF,mFAAmF,CACtF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAErB,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1D,SAAS,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACvD,MAAM,gBAAgB,EAAE,CAAC;IAC3B,CAAC;SAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,YAAY,EAAE,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,mCAAmC;QACnC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAC9B,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY;IACzB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAClD,MAAM,eAAe,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,uDAAuD;YACrD,iGAAiG,CACpG,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAChD,MAAM,YAAY,GAChB,OAAO,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1E,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEjE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,IAAI,MAAM,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACtC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,gCAAgC,IAAI,uBAAuB,CAC5D,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChB,OAAO;YACT,CAAC;YACD,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACpD,MAAM,iBAAiB,GACrB,aAAa,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAC7C,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QACpC,CAAC,CAAC,SAAS,CAAC;IAEhB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,UAAU,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,4BAA4B,UAAU,aAAa,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CACjG,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,kBAAkB,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,MAAM,SAAS,GACb,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IAE7D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,SAAS,GACb,YAAY,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAC3C,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,GAAG,CAAC;IAEV,IAAI,SAAS,KAAK,OAAO,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,gFAAgF;YAC9E,gFAAgF;YAChF,4CAA4C,CAC/C,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,0EAA0E;YACxE,yEAAyE;YACzE,6EAA6E;YAC7E,2DAA2D,CAC9D,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,gBAAgB,CAAC;QACvC,QAAQ,EAAE;YACR,OAAO,EAAE,eAAe;YACxB,IAAI,EAAE,YAAY;YAClB,GAAG,EAAE,WAAW;YAChB,GAAG,EAAE,WAAW;SACjB;QACD,UAAU;QACV,IAAI,EAAE,SAAS;QACf,SAAS;QACT,iBAAiB;KAClB,CAAC,CAAC;IAEH,MAAM,WAAW,CAAC,eAAe,EAAE,CAAC;IAEpC,iEAAiE;IACjE,2EAA2E;IAC3E,sEAAsE;IACtE,0EAA0E;IAC1E,0EAA0E;IAC1E,sEAAsE;IACtE,4EAA4E;IAC5E,qCAAqC;IACrC,MAAM,iBAAiB,GACrB,iBAAiB,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC;QAC/C,CAAC,CAAC,iBAAiB;QACnB,CAAC,CAAC,2BAA2B,CAAC;IAClC,MAAM,aAAa,GAAG,WAAW,CAAC,gBAAgB,EAAE,CAAC;IACrD,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9C,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAC1C,CAAC;IACF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6DAA6D,aAAa,CAAC,MAAM,GAAG;YAClF,YAAY,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB;YAChF,uBAAuB,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,oCAAoC;YACvF,oFAAoF;YACpF,6EAA6E;YAC7E,iCAAiC,CACpC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,IAAmB,EAAE;QACzC,MAAM,WAAW,CAAC,kBAAkB,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEhC,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAC9C,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC7B,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAE/B,2EAA2E;IAC3E,qEAAqE;IACrE,qDAAqD;IACrD,SAAS,cAAc,CAAC,GAAW;QACjC,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,CAAC;IAED,uBAAuB;IACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,OAAO,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,iEAAiE;YACjE,uEAAuE;YACvE,sEAAsE;YACtE,oEAAoE;YACpE,4DAA4D;YAC5D,oEAAoE;YACpE,mBAAmB;YACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,kDAAkD,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,MAAM,CAC1E,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;QACT,CAAC;QACD,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC;IAClC,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3C,IAAI,SAAS,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6DAA6D,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,MAAM,CACvF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;QACT,CAAC;QACD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;IAC3B,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,YAAY,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC;IACtE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;YACpD,UAAU,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAC1D,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC5D,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6EAA6E;QAC3E,4EAA4E;QAC5E,8EAA8E;QAC9E,+EAA+E;QAC/E,+DAA+D,CAClE,CAAC;IACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;IACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,CAAC,QAAQ,QAAQ,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;IAE1C,yBAAyB;IACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IAEvD,wBAAwB;IACxB,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,YAAY,CAAC;IAC/B,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,QAAQ,GAAG,GAAG,SAAS,wBAAwB,CAAC;IACtD,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,SAAS,CACb,QAAQ,EACR,IAAI,CAAC,SAAS,CACZ;QACE,GAAG,SAAS;QACZ,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,aAAa,EAAE,CAAC,CAAC,aAAa;SAC/B,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,EACD,IAAI,EACJ,CAAC,CACF,EACD,OAAO,CACR,CAAC;IACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,QAAQ,IAAI,CAAC,CAAC;IAExD,2BAA2B;IAC3B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../../src/proxy/gateway.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;
|
|
1
|
+
{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../../src/proxy/gateway.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAOnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAgEzD,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,gBAAgB,GAAG,MAAM,CA+CvE"}
|
package/dist/proxy/gateway.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
9
9
|
import { ListToolsRequestSchema, CallToolRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
10
10
|
import { z } from "zod";
|
|
11
|
+
import { VERSION } from "../version.js";
|
|
11
12
|
import { GetReportSchema, GetScoreSchema, ListScenariosSchema, ResetSessionSchema, DeleteSessionSchema, } from "../tools/schemas.js";
|
|
12
13
|
import { handleGetReport, handleGetScore, handleListScenarios, handleResetSession, handleDeleteSession, } from "../tools/handlers.js";
|
|
13
14
|
// session_id is required in the shared schemas (used by the multi-session
|
|
@@ -48,7 +49,7 @@ function isOwnTool(name) {
|
|
|
48
49
|
return name in OWN_TOOLS;
|
|
49
50
|
}
|
|
50
51
|
export function createProxyServer(interceptor) {
|
|
51
|
-
const server = new Server({ name: "cuonztech-agent-harness-proxy", version:
|
|
52
|
+
const server = new Server({ name: "cuonztech-agent-harness-proxy", version: VERSION }, { capabilities: { tools: {} } });
|
|
52
53
|
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
53
54
|
const upstreamDefs = interceptor.getUpstreamTools();
|
|
54
55
|
const ownDefs = Object.keys(OWN_TOOLS).map((name) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway.js","sourceRoot":"","sources":["../../src/proxy/gateway.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,0EAA0E;AAC1E,wEAAwE;AACxE,wEAAwE;AACxE,wEAAwE;AACxE,wEAAwE;AACxE,+CAA+C;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"gateway.js","sourceRoot":"","sources":["../../src/proxy/gateway.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,0EAA0E;AAC1E,wEAAwE;AACxE,wEAAwE;AACxE,wEAAwE;AACxE,wEAAwE;AACxE,+CAA+C;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EACL,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,0EAA0E;AAC1E,uEAAuE;AACvE,4EAA4E;AAC5E,kEAAkE;AAClE,SAAS,iBAAiB,CACxB,MAAoC;IAEpC,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,SAAS,GAAG;IAChB,UAAU,EAAE;QACV,KAAK,EAAE,kBAAkB;QACzB,WAAW,EACT,8FAA8F;QAChG,MAAM,EAAE,iBAAiB,CAAC,eAAe,CAAC;KAC3C;IACD,SAAS,EAAE;QACT,KAAK,EAAE,sBAAsB;QAC7B,WAAW,EACT,mKAAmK;QACrK,MAAM,EAAE,iBAAiB,CAAC,cAAc,CAAC;KAC1C;IACD,cAAc,EAAE;QACd,KAAK,EAAE,0BAA0B;QACjC,WAAW,EACT,oGAAoG;QACtG,MAAM,EAAE,mBAAmB;KAC5B;IACD,aAAa,EAAE;QACb,KAAK,EAAE,eAAe;QACtB,WAAW,EACT,2EAA2E;QAC7E,MAAM,EAAE,iBAAiB,CAAC,kBAAkB,CAAC;KAC9C;IACD,cAAc,EAAE;QACd,KAAK,EAAE,gBAAgB;QACvB,WAAW,EAAE,iDAAiD;QAC9D,MAAM,EAAE,iBAAiB,CAAC,mBAAmB,CAAC;KAC/C;CACO,CAAC;AAIX,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,IAAI,SAAS,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,WAA6B;IAC7D,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,+BAA+B,EAAE,OAAO,EAAE,OAAO,EAAE,EAC3D,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAChC,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,YAAY,GAAG,WAAW,CAAC,gBAAgB,EAAE,CAAC;QACpD,MAAM,OAAO,GAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAmB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACrE,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACnD,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,OAAO,IAAI,EAAE,CAA4B,CAAC;QAE5D,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,WAAW,GAAG;gBAClB,UAAU,EAAE,WAAW,CAAC,SAAS;gBACjC,GAAG,QAAQ;aACZ,CAAC;YACF,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,YAAY;oBACf,OAAO,eAAe,CAAC,WAAoD,CAAC,CAAC;gBAC/E,KAAK,WAAW;oBACd,OAAO,cAAc,CAAC,WAAmD,CAAC,CAAC;gBAC7E,KAAK,gBAAgB;oBACnB,OAAO,mBAAmB,EAAE,CAAC;gBAC/B,KAAK,eAAe;oBAClB,OAAO,kBAAkB,CAAC,WAAuD,CAAC,CAAC;gBACrF,KAAK,gBAAgB;oBACnB,OAAO,mBAAmB,CAAC,WAAwD,CAAC,CAAC;YACzF,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ghost-write.d.ts","sourceRoot":"","sources":["../../src/proxy/ghost-write.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAI3E,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,YAAY,EACrB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC5B,aAAa,
|
|
1
|
+
{"version":3,"file":"ghost-write.d.ts","sourceRoot":"","sources":["../../src/proxy/ghost-write.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAI3E,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,YAAY,EACrB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC5B,aAAa,CA6Df;AAED,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;CACpB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAC/C,gBAAgB,CAMlB;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,EAC5B,QAAQ,EAAE,MAAM,EAChB,gBAAgB,EAAE,OAAO,GACxB,UAAU,CAUZ"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { getScenarioById } from "../scenarios/f1-f5.js";
|
|
2
|
-
import { markGhostCommitted } from "../engine/state-machine.js";
|
|
2
|
+
import { markGhostCommitted, isWriteTool } from "../engine/state-machine.js";
|
|
3
3
|
export function decideChaosAction(session, callNumber, toolName, args) {
|
|
4
4
|
const noChaos = {
|
|
5
5
|
injectError: false,
|
|
@@ -9,18 +9,29 @@ export function decideChaosAction(session, callNumber, toolName, args) {
|
|
|
9
9
|
executeUpstream: true,
|
|
10
10
|
delayMs: 0,
|
|
11
11
|
};
|
|
12
|
+
const isWriteCall = isWriteTool(toolName, session.writeToolPatterns);
|
|
13
|
+
const priorWriteCalls = session.history.filter((r) => isWriteTool(r.toolName, session.writeToolPatterns)).length;
|
|
12
14
|
// Deterministic scenario mode
|
|
13
15
|
if (session.scenarioId) {
|
|
14
16
|
const scenario = getScenarioById(session.scenarioId);
|
|
15
|
-
if (scenario &&
|
|
17
|
+
if (scenario &&
|
|
18
|
+
scenario.triggerCondition({ callNumber, toolName, isWriteCall, priorWriteCalls })) {
|
|
16
19
|
const resp = scenario.simulatedResponse;
|
|
17
|
-
const isGhostWrite = resp.type === "timeout" &&
|
|
20
|
+
const isGhostWrite = resp.type === "timeout" && isWriteCall;
|
|
21
|
+
// F4 (duplicate dispatch) must also actually reach the upstream: it
|
|
22
|
+
// reports "success" to the agent, and against a real upstream that
|
|
23
|
+
// claim has to be true, or the proxy is fabricating a confirmation for
|
|
24
|
+
// a write that never happened (silent data loss if ever pointed at a
|
|
25
|
+
// production backend). F2/F3/F5 stay executeUpstream:false — those are
|
|
26
|
+
// genuine failures (rate-limit/malformed/500), so nothing should land
|
|
27
|
+
// upstream for them, matching what the agent is told.
|
|
28
|
+
const executeUpstream = isGhostWrite || resp.type === "success";
|
|
18
29
|
return {
|
|
19
30
|
injectError: true,
|
|
20
31
|
errorType: resp.type === "success" ? null : `[${resp.type.toUpperCase()}]`,
|
|
21
32
|
statusCode: resp.statusCode ?? 500,
|
|
22
33
|
errorBody: resp.body,
|
|
23
|
-
executeUpstream
|
|
34
|
+
executeUpstream,
|
|
24
35
|
delayMs: resp.delayMs ?? 0,
|
|
25
36
|
};
|
|
26
37
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ghost-write.js","sourceRoot":"","sources":["../../src/proxy/ghost-write.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"ghost-write.js","sourceRoot":"","sources":["../../src/proxy/ghost-write.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAW7E,MAAM,UAAU,iBAAiB,CAC/B,OAAqB,EACrB,UAAkB,EAClB,QAAgB,EAChB,IAA6B;IAE7B,MAAM,OAAO,GAAkB;QAC7B,WAAW,EAAE,KAAK;QAClB,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,IAAI;QACf,eAAe,EAAE,IAAI;QACrB,OAAO,EAAE,CAAC;KACX,CAAC;IAEF,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACrE,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACnD,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC,CACnD,CAAC,MAAM,CAAC;IAET,8BAA8B;IAC9B,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrD,IACE,QAAQ;YACR,QAAQ,CAAC,gBAAgB,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,EACjF,CAAC;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,iBAAiB,CAAC;YACxC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,WAAW,CAAC;YAC5D,oEAAoE;YACpE,mEAAmE;YACnE,uEAAuE;YACvE,qEAAqE;YACrE,uEAAuE;YACvE,sEAAsE;YACtE,sDAAsD;YACtD,MAAM,eAAe,GAAG,YAAY,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC;YAEhE,OAAO;gBACL,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG;gBAC1E,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,GAAG;gBAClC,SAAS,EAAE,IAAI,CAAC,IAAI;gBACpB,eAAe;gBACf,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC;aAC3B,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,yCAAyC;IACzC,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;QAClE,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,kBAAkB;YAC7B,UAAU,EAAE,GAAG;YACf,SAAS,EAAE;gBACT,KAAK,EAAE,aAAa;gBACpB,OAAO,EAAE,kCAAkC;aAC5C;YACD,eAAe,EAAE,KAAK;YACtB,OAAO,EAAE,CAAC;SACX,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD,MAAM,UAAU,gBAAgB,CAC9B,OAAqB,EACrB,IAA6B,EAC7B,gBAAgD;IAEhD,MAAM,GAAG,GAAI,IAAI,CAAC,iBAAiB,CAAY,IAAI,IAAI,CAAC;IACxD,IAAI,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAC5B,kBAAkB,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,UAAkB,EAClB,QAAgB,EAChB,IAA6B,EAC7B,aAA4B,EAC5B,QAAgB,EAChB,gBAAyB;IAEzB,OAAO;QACL,UAAU;QACV,QAAQ;QACR,IAAI;QACJ,aAAa;QACb,QAAQ;QACR,gBAAgB;QAChB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;AACJ,CAAC"}
|
|
@@ -5,11 +5,13 @@ export interface ProxyConfig {
|
|
|
5
5
|
scenarioId?: string;
|
|
6
6
|
mode?: "deterministic" | "chaos";
|
|
7
7
|
errorRate?: number;
|
|
8
|
+
writeToolPatterns?: string[];
|
|
8
9
|
}
|
|
9
10
|
export interface UpstreamConfig {
|
|
10
11
|
command: string;
|
|
11
12
|
args?: string[];
|
|
12
13
|
cwd?: string;
|
|
14
|
+
env?: Record<string, string>;
|
|
13
15
|
}
|
|
14
16
|
export declare class ProxyInterceptor {
|
|
15
17
|
private upstreamClient;
|
|
@@ -24,10 +26,7 @@ export declare class ProxyInterceptor {
|
|
|
24
26
|
getUpstreamTools(): Tool[];
|
|
25
27
|
getSession(): SessionState;
|
|
26
28
|
interceptToolCall(toolName: string, args: Record<string, unknown>): Promise<{
|
|
27
|
-
content: Array<
|
|
28
|
-
type: "text";
|
|
29
|
-
text: string;
|
|
30
|
-
}>;
|
|
29
|
+
content: Array<Record<string, unknown>>;
|
|
31
30
|
isError: boolean;
|
|
32
31
|
}>;
|
|
33
32
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interceptor.d.ts","sourceRoot":"","sources":["../../src/proxy/interceptor.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"interceptor.d.ts","sourceRoot":"","sources":["../../src/proxy/interceptor.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAE/D,OAAO,EAKL,KAAK,YAAY,EAClB,MAAM,4BAA4B,CAAC;AAWpC,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,cAAc,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IAOb,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,iBAAiB,CAA0B;IACnD,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,aAAa,CAAc;IAEnC,YAAY,MAAM,EAAE,WAAW,EAQ9B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAEK,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAyBrC;IAEK,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAKxC;IAED,gBAAgB,IAAI,IAAI,EAAE,CAEzB;IAED,UAAU,IAAI,YAAY,CAEzB;IAEK,iBAAiB,CACrB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC5B,OAAO,CAAC;QACT,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACxC,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAsID;CACF"}
|
|
@@ -5,8 +5,12 @@
|
|
|
5
5
|
// anhand `getUpstreamTools()`).
|
|
6
6
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
7
7
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
8
|
+
import { VERSION } from "../version.js";
|
|
8
9
|
import { createSession, incrementCall, addRecord, } from "../engine/state-machine.js";
|
|
9
10
|
import { decideChaosAction, handleGhostWrite, buildCallRecord, } from "./ghost-write.js";
|
|
11
|
+
function sleep(ms) {
|
|
12
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
13
|
+
}
|
|
10
14
|
export class ProxyInterceptor {
|
|
11
15
|
upstreamClient = null;
|
|
12
16
|
upstreamTransport = null;
|
|
@@ -15,7 +19,7 @@ export class ProxyInterceptor {
|
|
|
15
19
|
upstreamTools = [];
|
|
16
20
|
constructor(config) {
|
|
17
21
|
this.config = config;
|
|
18
|
-
this.session = createSession(config.scenarioId ?? null, config.mode ?? "deterministic", config.errorRate ?? 0.0);
|
|
22
|
+
this.session = createSession(config.scenarioId ?? null, config.mode ?? "deterministic", config.errorRate ?? 0.0, config.writeToolPatterns);
|
|
19
23
|
}
|
|
20
24
|
get sessionId() {
|
|
21
25
|
return this.session.sessionId;
|
|
@@ -25,9 +29,16 @@ export class ProxyInterceptor {
|
|
|
25
29
|
command: this.config.upstream.command,
|
|
26
30
|
args: this.config.upstream.args,
|
|
27
31
|
cwd: this.config.upstream.cwd,
|
|
32
|
+
// Inherit the full parent environment (same as any normal child
|
|
33
|
+
// process spawn) instead of the SDK's restrictive default allowlist,
|
|
34
|
+
// then layer --upstream-env on top for additions/overrides.
|
|
35
|
+
env: {
|
|
36
|
+
...process.env,
|
|
37
|
+
...this.config.upstream.env,
|
|
38
|
+
},
|
|
28
39
|
stderr: "pipe",
|
|
29
40
|
});
|
|
30
|
-
this.upstreamClient = new Client({ name: "cuonztech-harness-proxy", version:
|
|
41
|
+
this.upstreamClient = new Client({ name: "cuonztech-harness-proxy", version: VERSION }, { capabilities: {} });
|
|
31
42
|
await this.upstreamClient.connect(this.upstreamTransport);
|
|
32
43
|
// Discover upstream tools (full definitions, including inputSchema, so
|
|
33
44
|
// the gateway can advertise them to its own client unchanged)
|
|
@@ -54,9 +65,12 @@ export class ProxyInterceptor {
|
|
|
54
65
|
// Decide chaos action
|
|
55
66
|
const decision = decideChaosAction(this.session, callNumber, toolName, mergedArgs);
|
|
56
67
|
let upstreamResponse = null;
|
|
68
|
+
let upstreamContent = null;
|
|
69
|
+
let upstreamIsError = false;
|
|
57
70
|
let upstreamExecuted = false;
|
|
71
|
+
// responseText only ever feeds the audit trail (buildCallRecord/history),
|
|
72
|
+
// never the agent — see agentContent below.
|
|
58
73
|
let responseText = "";
|
|
59
|
-
let statusCode = 200;
|
|
60
74
|
let injectedError = null;
|
|
61
75
|
if (decision.executeUpstream && this.upstreamClient) {
|
|
62
76
|
// Forward to upstream
|
|
@@ -65,9 +79,10 @@ export class ProxyInterceptor {
|
|
|
65
79
|
name: toolName,
|
|
66
80
|
arguments: mergedArgs,
|
|
67
81
|
});
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
82
|
+
upstreamContent = result.content;
|
|
83
|
+
upstreamIsError = Boolean(result.isError);
|
|
84
|
+
const textPart = upstreamContent.find((c) => c.type === "text");
|
|
85
|
+
responseText = textPart?.text ?? JSON.stringify(upstreamContent);
|
|
71
86
|
upstreamExecuted = true;
|
|
72
87
|
try {
|
|
73
88
|
upstreamResponse = JSON.parse(responseText);
|
|
@@ -84,27 +99,59 @@ export class ProxyInterceptor {
|
|
|
84
99
|
injectedError = "[UPSTREAM_ERROR]";
|
|
85
100
|
}
|
|
86
101
|
}
|
|
87
|
-
//
|
|
102
|
+
// A genuine simulated failure (timeout/rate_limit/malformed/error) must
|
|
103
|
+
// replace whatever the agent sees, even if the upstream really executed
|
|
104
|
+
// (F1's ghost-write). A "success"-typed match (F4) is NOT a failure —
|
|
105
|
+
// once it actually ran upstream above, the agent must see that REAL
|
|
106
|
+
// response, not a canned one, or the proxy is lying about what happened.
|
|
107
|
+
const isSimulatedFailure = decision.injectError && decision.errorType !== null;
|
|
88
108
|
let isGhostWrite = false;
|
|
89
|
-
|
|
109
|
+
let agentContent;
|
|
110
|
+
let agentIsError;
|
|
111
|
+
if (isSimulatedFailure) {
|
|
112
|
+
// Honor the scenario's configured delay before replying — previously
|
|
113
|
+
// decision.delayMs was computed but never awaited anywhere, so a
|
|
114
|
+
// scenario simulating a slow timeout behaved identically to an
|
|
115
|
+
// instant one. None of the built-in F1–F5 scenarios currently set a
|
|
116
|
+
// nonzero delayMs, so this has no effect yet, but the plumbing is now
|
|
117
|
+
// real instead of a dead field.
|
|
118
|
+
if (decision.delayMs > 0) {
|
|
119
|
+
await sleep(decision.delayMs);
|
|
120
|
+
}
|
|
90
121
|
injectedError = decision.errorType;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
122
|
+
const errorText = typeof decision.errorBody === "string"
|
|
123
|
+
? decision.errorBody
|
|
124
|
+
: JSON.stringify(decision.errorBody);
|
|
125
|
+
responseText = errorText;
|
|
126
|
+
agentContent = [{ type: "text", text: errorText }];
|
|
127
|
+
agentIsError = true;
|
|
96
128
|
isGhostWrite = decision.executeUpstream && upstreamExecuted && Boolean(key);
|
|
97
129
|
}
|
|
130
|
+
else if (upstreamExecuted) {
|
|
131
|
+
// Real passthrough: forward the upstream's actual content unchanged
|
|
132
|
+
// (all blocks — text, images, structured content — not just the first
|
|
133
|
+
// text block) and its real isError, instead of wrapping it in an
|
|
134
|
+
// audit envelope. Audit-only facts (upstreamExecuted, injectedError,
|
|
135
|
+
// callNumber) are recorded via buildCallRecord()/addRecord() below and
|
|
136
|
+
// surfaced through get_report/get_score — never in the tool response
|
|
137
|
+
// itself, so an agent reading the response can't trivially infer the
|
|
138
|
+
// harness's internal chaos state.
|
|
139
|
+
agentContent = upstreamContent ?? [{ type: "text", text: responseText }];
|
|
140
|
+
agentIsError = upstreamIsError;
|
|
141
|
+
}
|
|
142
|
+
else if (injectedError === "[UPSTREAM_ERROR]") {
|
|
143
|
+
agentContent = [{ type: "text", text: responseText }];
|
|
144
|
+
agentIsError = true;
|
|
145
|
+
}
|
|
98
146
|
else {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
}
|
|
147
|
+
// No scenario triggered and no real upstream call was attempted
|
|
148
|
+
// (shouldn't normally happen once connected) — synthetic ok mock.
|
|
149
|
+
responseText = JSON.stringify({
|
|
150
|
+
result: "ok",
|
|
151
|
+
timestamp: new Date().toISOString(),
|
|
152
|
+
});
|
|
153
|
+
agentContent = [{ type: "text", text: responseText }];
|
|
154
|
+
agentIsError = false;
|
|
108
155
|
}
|
|
109
156
|
// Record the call
|
|
110
157
|
const record = buildCallRecord(callNumber, toolName, mergedArgs, injectedError, responseText, upstreamExecuted);
|
|
@@ -118,26 +165,8 @@ export class ProxyInterceptor {
|
|
|
118
165
|
handleGhostWrite(this.session, mergedArgs, upstreamResponse);
|
|
119
166
|
}
|
|
120
167
|
return {
|
|
121
|
-
content:
|
|
122
|
-
|
|
123
|
-
type: "text",
|
|
124
|
-
text: JSON.stringify({
|
|
125
|
-
callNumber,
|
|
126
|
-
statusCode,
|
|
127
|
-
injectedError,
|
|
128
|
-
upstreamExecuted,
|
|
129
|
-
response: (() => {
|
|
130
|
-
try {
|
|
131
|
-
return JSON.parse(responseText);
|
|
132
|
-
}
|
|
133
|
-
catch {
|
|
134
|
-
return responseText;
|
|
135
|
-
}
|
|
136
|
-
})(),
|
|
137
|
-
}, null, 2),
|
|
138
|
-
},
|
|
139
|
-
],
|
|
140
|
-
isError: statusCode >= 400,
|
|
168
|
+
content: agentContent,
|
|
169
|
+
isError: agentIsError,
|
|
141
170
|
};
|
|
142
171
|
}
|
|
143
172
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interceptor.js","sourceRoot":"","sources":["../../src/proxy/interceptor.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,+EAA+E;AAC/E,8EAA8E;AAC9E,4EAA4E;AAC5E,gCAAgC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAGjF,OAAO,EACL,aAAa,EAEb,aAAa,EACb,SAAS,GAEV,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,GAChB,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"interceptor.js","sourceRoot":"","sources":["../../src/proxy/interceptor.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,+EAA+E;AAC/E,8EAA8E;AAC9E,4EAA4E;AAC5E,gCAAgC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAGjF,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EACL,aAAa,EAEb,aAAa,EACb,SAAS,GAEV,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAE1B,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAuBD,MAAM,OAAO,gBAAgB;IACnB,cAAc,GAAkB,IAAI,CAAC;IACrC,iBAAiB,GAAqB,IAAI,CAAC;IAC3C,OAAO,CAAe;IACtB,MAAM,CAAc;IACpB,aAAa,GAAW,EAAE,CAAC;IAEnC,YAAY,MAAmB;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,aAAa,CAC1B,MAAM,CAAC,UAAU,IAAI,IAAI,EACzB,MAAM,CAAC,IAAI,IAAI,eAAe,EAC9B,MAAM,CAAC,SAAS,IAAI,GAAG,EACvB,MAAM,CAAC,iBAAiB,CACzB,CAAC;IACJ,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,iBAAiB,GAAG,IAAI,oBAAoB,CAAC;YAChD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO;YACrC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI;YAC/B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG;YAC7B,gEAAgE;YAChE,qEAAqE;YACrE,4DAA4D;YAC5D,GAAG,EAAE;gBACH,GAAI,OAAO,CAAC,GAA8B;gBAC1C,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG;aAC5B;YACD,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,GAAG,IAAI,MAAM,CAC9B,EAAE,IAAI,EAAE,yBAAyB,EAAE,OAAO,EAAE,OAAO,EAAE,EACrD,EAAE,YAAY,EAAE,EAAE,EAAE,CACrB,CAAC;QACF,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE1D,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC;QACpD,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,QAAgB,EAChB,IAA6B;QAK7B,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEzD,kCAAkC;QAClC,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,iBAAiB,CAAuB,CAAC;QAE1D,sBAAsB;QACtB,MAAM,QAAQ,GAAG,iBAAiB,CAChC,IAAI,CAAC,OAAO,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,CACX,CAAC;QAEF,IAAI,gBAAgB,GAAmC,IAAI,CAAC;QAC5D,IAAI,eAAe,GAA0C,IAAI,CAAC;QAClE,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,0EAA0E;QAC1E,4CAA4C;QAC5C,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,aAAa,GAAkB,IAAI,CAAC;QAExC,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACpD,sBAAsB;YACtB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;oBAChD,IAAI,EAAE,QAAQ;oBACd,SAAS,EAAE,UAAU;iBACtB,CAAC,CAAC;gBACH,eAAe,GAAG,MAAM,CAAC,OAAyC,CAAC;gBACnE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC1C,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAEjD,CAAC;gBACd,YAAY,GAAG,QAAQ,EAAE,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;gBACjE,gBAAgB,GAAG,IAAI,CAAC;gBAExB,IAAI,CAAC;oBACH,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC9C,CAAC;gBAAC,MAAM,CAAC;oBACP,gBAAgB,GAAG,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC;gBAC3C,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;oBAC5B,KAAK,EAAE,gBAAgB;oBACvB,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1D,CAAC,CAAC;gBACH,aAAa,GAAG,kBAAkB,CAAC;YACrC,CAAC;QACH,CAAC;QAED,wEAAwE;QACxE,wEAAwE;QACxE,sEAAsE;QACtE,oEAAoE;QACpE,yEAAyE;QACzE,MAAM,kBAAkB,GAAG,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC;QAE/E,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,YAA4C,CAAC;QACjD,IAAI,YAAqB,CAAC;QAE1B,IAAI,kBAAkB,EAAE,CAAC;YACvB,qEAAqE;YACrE,iEAAiE;YACjE,+DAA+D;YAC/D,oEAAoE;YACpE,sEAAsE;YACtE,gCAAgC;YAChC,IAAI,QAAQ,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;YAED,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC;YACnC,MAAM,SAAS,GACb,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ;gBACpC,CAAC,CAAC,QAAQ,CAAC,SAAS;gBACpB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzC,YAAY,GAAG,SAAS,CAAC;YACzB,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YACnD,YAAY,GAAG,IAAI,CAAC;YAEpB,YAAY,GAAG,QAAQ,CAAC,eAAe,IAAI,gBAAgB,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9E,CAAC;aAAM,IAAI,gBAAgB,EAAE,CAAC;YAC5B,oEAAoE;YACpE,sEAAsE;YACtE,iEAAiE;YACjE,qEAAqE;YACrE,uEAAuE;YACvE,qEAAqE;YACrE,qEAAqE;YACrE,kCAAkC;YAClC,YAAY,GAAG,eAAe,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;YACzE,YAAY,GAAG,eAAe,CAAC;QACjC,CAAC;aAAM,IAAI,aAAa,KAAK,kBAAkB,EAAE,CAAC;YAChD,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;YACtD,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,gEAAgE;YAChE,kEAAkE;YAClE,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;gBAC5B,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;YACH,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;YACtD,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,kBAAkB;QAClB,MAAM,MAAM,GAAG,eAAe,CAC5B,UAAU,EACV,QAAQ,EACR,UAAU,EACV,aAAa,EACb,YAAY,EACZ,gBAAgB,CACjB,CAAC;QACF,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE1C,wEAAwE;QACxE,sEAAsE;QACtE,sEAAsE;QACtE,qEAAqE;QACrE,iDAAiD;QACjD,IAAI,YAAY,EAAE,CAAC;YACjB,gBAAgB,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO;YACL,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,YAAY;SACtB,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"f1-f5.d.ts","sourceRoot":"","sources":["../../src/scenarios/f1-f5.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAGrD,eAAO,MAAM,sBAAsB,EAAE,
|
|
1
|
+
{"version":3,"file":"f1-f5.d.ts","sourceRoot":"","sources":["../../src/scenarios/f1-f5.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAGrD,eAAO,MAAM,sBAAsB,EAAE,kBAmBpC,CAAC;AAGF,eAAO,MAAM,aAAa,EAAE,kBAiB3B,CAAC;AAGF,eAAO,MAAM,iBAAiB,EAAE,kBAa/B,CAAC;AAGF,eAAO,MAAM,qBAAqB,EAAE,kBA0BnC,CAAC;AAGF,eAAO,MAAM,4BAA4B,EAAE,kBAgB1C,CAAC;AAEF,eAAO,MAAM,aAAa,EAAE,kBAAkB,EAM7C,CAAC;AAEF,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS,CAE1E"}
|
package/dist/scenarios/f1-f5.js
CHANGED
|
@@ -3,7 +3,11 @@ export const F1_TIMEOUT_FIRST_WRITE = {
|
|
|
3
3
|
id: "F1",
|
|
4
4
|
name: "Timeout on First Write",
|
|
5
5
|
description: "Simuliert einen Timeout beim ersten Schreibversuch. Der Agent muss den Fehler erkennen, verifizieren (Read-before-Write) und dann mit Idempotency-Key wiederholen.",
|
|
6
|
-
|
|
6
|
+
// Fires on the agent's first WRITE call specifically, regardless of its
|
|
7
|
+
// position in the overall call sequence — an agent that reads first, then
|
|
8
|
+
// writes, must still see the timeout on that write (previously gated on
|
|
9
|
+
// callNumber === 1, so a cautious read-first agent never triggered F1 at all).
|
|
10
|
+
triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls === 0 && isWriteCall,
|
|
7
11
|
simulatedResponse: {
|
|
8
12
|
type: "timeout",
|
|
9
13
|
statusCode: 408,
|
|
@@ -17,7 +21,7 @@ export const F2_RATE_LIMIT = {
|
|
|
17
21
|
id: "F2",
|
|
18
22
|
name: "Rate Limit (429)",
|
|
19
23
|
description: "Simuliert einen HTTP 429 Rate-Limit beim zweiten Aufruf. Der Agent muss Backoff respektieren und nicht sofort retry-en.",
|
|
20
|
-
triggerCondition: (callNumber) => callNumber === 2,
|
|
24
|
+
triggerCondition: ({ callNumber }) => callNumber === 2,
|
|
21
25
|
simulatedResponse: {
|
|
22
26
|
type: "rate_limit",
|
|
23
27
|
statusCode: 429,
|
|
@@ -34,7 +38,7 @@ export const F3_MALFORMED_JSON = {
|
|
|
34
38
|
id: "F3",
|
|
35
39
|
name: "Malformed JSON Response",
|
|
36
40
|
description: "Liefert ein defektes JSON zurück. Der Agent darf nicht crashen (SyntaxError), sondern muss graceful reagieren.",
|
|
37
|
-
triggerCondition: (callNumber) => callNumber === 1,
|
|
41
|
+
triggerCondition: ({ callNumber }) => callNumber === 1,
|
|
38
42
|
simulatedResponse: {
|
|
39
43
|
type: "malformed",
|
|
40
44
|
statusCode: 200,
|
|
@@ -46,8 +50,16 @@ export const F3_MALFORMED_JSON = {
|
|
|
46
50
|
export const F4_DUPLICATE_DISPATCH = {
|
|
47
51
|
id: "F4",
|
|
48
52
|
name: "Duplicate Dispatch (Ghost-Write)",
|
|
49
|
-
description: "Beide Aufrufe erhalten Erfolg.
|
|
50
|
-
|
|
53
|
+
description: "Beide Aufrufe erhalten Erfolg. Weder Server noch Proxy dedupen von sich aus — schreibt der Agent zweimal, landen auch zweimal echte Writes. Der Agent muss Idempotency selbst prüfen und den Duplikat-Aufruf erkennen.",
|
|
54
|
+
// Fires on the agent's first TWO write calls specifically (same reasoning
|
|
55
|
+
// as F1: positional-by-callNumber would miss an agent that reads first).
|
|
56
|
+
triggerCondition: ({ priorWriteCalls, isWriteCall }) => priorWriteCalls <= 1 && isWriteCall,
|
|
57
|
+
// Only used by the non-proxy session/benchmark simulation (tools/handlers.ts),
|
|
58
|
+
// which has no real upstream to call. In proxy mode (proxy/ghost-write.ts +
|
|
59
|
+
// proxy/interceptor.ts), "success"-typed scenarios now actually execute
|
|
60
|
+
// against the real upstream and forward its real response/id instead of
|
|
61
|
+
// this canned body — a fabricated transaction_id for a write that never
|
|
62
|
+
// happened would be a real data-loss risk against a production backend.
|
|
51
63
|
simulatedResponse: {
|
|
52
64
|
type: "success",
|
|
53
65
|
statusCode: 200,
|
|
@@ -64,7 +76,7 @@ export const F5_SERVER_ERROR_THEN_SUCCESS = {
|
|
|
64
76
|
id: "F5",
|
|
65
77
|
name: "Server Error (500) then Success",
|
|
66
78
|
description: "Erster Aufruf: HTTP 500. Zweiter Aufruf: Erfolg. Der Agent muss den Fehler erkennen, einmal retry-en und dann stoppen.",
|
|
67
|
-
triggerCondition: (callNumber) => callNumber === 1,
|
|
79
|
+
triggerCondition: ({ callNumber }) => callNumber === 1,
|
|
68
80
|
simulatedResponse: {
|
|
69
81
|
type: "error",
|
|
70
82
|
statusCode: 500,
|