@inspectr/mcplab 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +762 -0
- package/dist/app/android-chrome-192x192.png +0 -0
- package/dist/app/android-chrome-512x512.png +0 -0
- package/dist/app/apple-touch-icon.png +0 -0
- package/dist/app/assets/index-DT-Z4AVG.js +249 -0
- package/dist/app/assets/index-EP4WAY8u.css +1 -0
- package/dist/app/favicon-16x16.png +0 -0
- package/dist/app/favicon-32x32.png +0 -0
- package/dist/app/favicon.svg +13 -0
- package/dist/app/index.html +26 -0
- package/dist/app/inspectr_logo_color.svg +9 -0
- package/dist/app/mcp.png +0 -0
- package/dist/app/mcp.svg +1 -0
- package/dist/app/robots.txt +14 -0
- package/dist/app/site.webmanifest +1 -0
- package/dist/app-server/app-context.d.ts +164 -0
- package/dist/app-server/app-context.d.ts.map +1 -0
- package/dist/app-server/app-context.js +2 -0
- package/dist/app-server/app-context.js.map +1 -0
- package/dist/app-server/assistant-common.d.ts +41 -0
- package/dist/app-server/assistant-common.d.ts.map +1 -0
- package/dist/app-server/assistant-common.js +104 -0
- package/dist/app-server/assistant-common.js.map +1 -0
- package/dist/app-server/config-store.d.ts +15 -0
- package/dist/app-server/config-store.d.ts.map +1 -0
- package/dist/app-server/config-store.js +67 -0
- package/dist/app-server/config-store.js.map +1 -0
- package/dist/app-server/dev-mcp.d.ts +6 -0
- package/dist/app-server/dev-mcp.d.ts.map +1 -0
- package/dist/app-server/dev-mcp.js +71 -0
- package/dist/app-server/dev-mcp.js.map +1 -0
- package/dist/app-server/evals-routes.d.ts +22 -0
- package/dist/app-server/evals-routes.d.ts.map +1 -0
- package/dist/app-server/evals-routes.js +135 -0
- package/dist/app-server/evals-routes.js.map +1 -0
- package/dist/app-server/http.d.ts +5 -0
- package/dist/app-server/http.d.ts.map +1 -0
- package/dist/app-server/http.js +31 -0
- package/dist/app-server/http.js.map +1 -0
- package/dist/app-server/index.d.ts +3 -0
- package/dist/app-server/index.d.ts.map +1 -0
- package/dist/app-server/index.js +2 -0
- package/dist/app-server/index.js.map +1 -0
- package/dist/app-server/jobs.d.ts +15 -0
- package/dist/app-server/jobs.d.ts.map +1 -0
- package/dist/app-server/jobs.js +11 -0
- package/dist/app-server/jobs.js.map +1 -0
- package/dist/app-server/libraries-store.d.ts +15 -0
- package/dist/app-server/libraries-store.d.ts.map +1 -0
- package/dist/app-server/libraries-store.js +61 -0
- package/dist/app-server/libraries-store.js.map +1 -0
- package/dist/app-server/markdown-reports.d.ts +12 -0
- package/dist/app-server/markdown-reports.d.ts.map +1 -0
- package/dist/app-server/markdown-reports.js +145 -0
- package/dist/app-server/markdown-reports.js.map +1 -0
- package/dist/app-server/oauth-debugger-domain.d.ts +230 -0
- package/dist/app-server/oauth-debugger-domain.d.ts.map +1 -0
- package/dist/app-server/oauth-debugger-domain.js +1098 -0
- package/dist/app-server/oauth-debugger-domain.js.map +1 -0
- package/dist/app-server/oauth-debugger.d.ts +20 -0
- package/dist/app-server/oauth-debugger.d.ts.map +1 -0
- package/dist/app-server/oauth-debugger.js +193 -0
- package/dist/app-server/oauth-debugger.js.map +1 -0
- package/dist/app-server/provider-models.d.ts +8 -0
- package/dist/app-server/provider-models.d.ts.map +1 -0
- package/dist/app-server/provider-models.js +60 -0
- package/dist/app-server/provider-models.js.map +1 -0
- package/dist/app-server/result-assistant-domain.d.ts +87 -0
- package/dist/app-server/result-assistant-domain.d.ts.map +1 -0
- package/dist/app-server/result-assistant-domain.js +212 -0
- package/dist/app-server/result-assistant-domain.js.map +1 -0
- package/dist/app-server/result-assistant.d.ts +22 -0
- package/dist/app-server/result-assistant.d.ts.map +1 -0
- package/dist/app-server/result-assistant.js +328 -0
- package/dist/app-server/result-assistant.js.map +1 -0
- package/dist/app-server/router.d.ts +4 -0
- package/dist/app-server/router.d.ts.map +1 -0
- package/dist/app-server/router.js +374 -0
- package/dist/app-server/router.js.map +1 -0
- package/dist/app-server/runs-routes.d.ts +44 -0
- package/dist/app-server/runs-routes.d.ts.map +1 -0
- package/dist/app-server/runs-routes.js +555 -0
- package/dist/app-server/runs-routes.js.map +1 -0
- package/dist/app-server/runs-store.d.ts +23 -0
- package/dist/app-server/runs-store.d.ts.map +1 -0
- package/dist/app-server/runs-store.js +84 -0
- package/dist/app-server/runs-store.js.map +1 -0
- package/dist/app-server/scenario-assistant-domain.d.ts +162 -0
- package/dist/app-server/scenario-assistant-domain.d.ts.map +1 -0
- package/dist/app-server/scenario-assistant-domain.js +269 -0
- package/dist/app-server/scenario-assistant-domain.js.map +1 -0
- package/dist/app-server/scenario-assistant.d.ts +29 -0
- package/dist/app-server/scenario-assistant.d.ts.map +1 -0
- package/dist/app-server/scenario-assistant.js +246 -0
- package/dist/app-server/scenario-assistant.js.map +1 -0
- package/dist/app-server/settings-store.d.ts +4 -0
- package/dist/app-server/settings-store.d.ts.map +1 -0
- package/dist/app-server/settings-store.js +32 -0
- package/dist/app-server/settings-store.js.map +1 -0
- package/dist/app-server/snapshots-routes.d.ts +24 -0
- package/dist/app-server/snapshots-routes.d.ts.map +1 -0
- package/dist/app-server/snapshots-routes.js +82 -0
- package/dist/app-server/snapshots-routes.js.map +1 -0
- package/dist/app-server/static-serving.d.ts +17 -0
- package/dist/app-server/static-serving.d.ts.map +1 -0
- package/dist/app-server/static-serving.js +64 -0
- package/dist/app-server/static-serving.js.map +1 -0
- package/dist/app-server/store-utils.d.ts +5 -0
- package/dist/app-server/store-utils.d.ts.map +1 -0
- package/dist/app-server/store-utils.js +26 -0
- package/dist/app-server/store-utils.js.map +1 -0
- package/dist/app-server/tool-analysis-domain.d.ts +146 -0
- package/dist/app-server/tool-analysis-domain.d.ts.map +1 -0
- package/dist/app-server/tool-analysis-domain.js +556 -0
- package/dist/app-server/tool-analysis-domain.js.map +1 -0
- package/dist/app-server/tool-analysis-storage.d.ts +41 -0
- package/dist/app-server/tool-analysis-storage.d.ts.map +1 -0
- package/dist/app-server/tool-analysis-storage.js +110 -0
- package/dist/app-server/tool-analysis-storage.js.map +1 -0
- package/dist/app-server/tool-analysis.d.ts +22 -0
- package/dist/app-server/tool-analysis.d.ts.map +1 -0
- package/dist/app-server/tool-analysis.js +271 -0
- package/dist/app-server/tool-analysis.js.map +1 -0
- package/dist/app-server/types.d.ts +28 -0
- package/dist/app-server/types.d.ts.map +1 -0
- package/dist/app-server/types.js +2 -0
- package/dist/app-server/types.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +544 -0
- package/dist/cli.js.map +1 -0
- package/dist/snapshot.d.ts +80 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +401 -0
- package/dist/snapshot.js.map +1 -0
- package/package.json +55 -0
|
@@ -0,0 +1,555 @@
|
|
|
1
|
+
import { existsSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, join } from 'node:path';
|
|
3
|
+
import { McpClientManager, loadConfig, runAll } from '@inspectr/mcplab-core';
|
|
4
|
+
import { renderReport } from '@inspectr/mcplab-reporting';
|
|
5
|
+
export async function handleRunsRoutes(params) {
|
|
6
|
+
const { req, res, pathname, method, settings, jobs, activeJobState, deps } = params;
|
|
7
|
+
const { parseBody, asJson, addJobEvent, sendSseEvent, ensureInsideRoot, listRuns, getRunResults, getScenarioRunTraceRecords, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, loadSnapshot, compareRunToSnapshot, applySnapshotPolicyToRunResult, readLibraries, pickDefaultAssistantAgentName, resolveAssistantAgentFromLibraries, chatWithAgent, pkgVersion } = deps;
|
|
8
|
+
if (pathname === '/api/runs' && method === 'GET') {
|
|
9
|
+
asJson(res, 200, listRuns(settings.runsDir));
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
if (pathname.startsWith('/api/runs/') && pathname.endsWith('/trace') && method === 'GET') {
|
|
13
|
+
const runId = pathname.split('/')[3];
|
|
14
|
+
asJson(res, 200, { runId, records: getScenarioRunTraceRecords(runId, settings.runsDir) });
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
if (pathname.startsWith('/api/runs/jobs/') && pathname.endsWith('/events') && method === 'GET') {
|
|
18
|
+
const jobId = pathname.split('/')[4];
|
|
19
|
+
const job = jobs.get(jobId);
|
|
20
|
+
if (!job) {
|
|
21
|
+
asJson(res, 404, { error: 'Job not found' });
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
res.statusCode = 200;
|
|
25
|
+
res.setHeader('content-type', 'text/event-stream');
|
|
26
|
+
res.setHeader('cache-control', 'no-cache');
|
|
27
|
+
res.setHeader('connection', 'keep-alive');
|
|
28
|
+
if ('flushHeaders' in res && typeof res.flushHeaders === 'function') {
|
|
29
|
+
res.flushHeaders();
|
|
30
|
+
}
|
|
31
|
+
for (const event of job.events)
|
|
32
|
+
sendSseEvent(res, event);
|
|
33
|
+
if (job.status !== 'running') {
|
|
34
|
+
res.end();
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
job.clients.add(res);
|
|
38
|
+
req.on('close', () => {
|
|
39
|
+
job.clients.delete(res);
|
|
40
|
+
});
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
if (pathname.startsWith('/api/runs/jobs/') && pathname.endsWith('/stop') && method === 'POST') {
|
|
44
|
+
const jobId = pathname.split('/')[4];
|
|
45
|
+
const job = jobs.get(jobId);
|
|
46
|
+
if (!job) {
|
|
47
|
+
asJson(res, 404, { error: 'Job not found' });
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
if (job.status !== 'running') {
|
|
51
|
+
asJson(res, 200, { ok: true, status: job.status });
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
54
|
+
job.abortController.abort();
|
|
55
|
+
job.status = 'stopped';
|
|
56
|
+
activeJobState.set(null);
|
|
57
|
+
asJson(res, 200, { ok: true, status: 'stopped' });
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
if (pathname === '/api/runs' && method === 'POST') {
|
|
61
|
+
if (activeJobState.get()) {
|
|
62
|
+
asJson(res, 409, { error: 'Another run is already active', jobId: activeJobState.get() });
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
const body = (await parseBody(req));
|
|
66
|
+
const configPathRaw = String(body.configPath ?? '');
|
|
67
|
+
const runsPerScenario = Number(body.runsPerScenario ?? 1);
|
|
68
|
+
const scenarioId = body.scenarioId ? String(body.scenarioId) : undefined;
|
|
69
|
+
const scenarioIds = Array.isArray(body.scenarioIds)
|
|
70
|
+
? body.scenarioIds.map((id) => String(id).trim()).filter(Boolean)
|
|
71
|
+
: undefined;
|
|
72
|
+
const requestedAgents = Array.isArray(body.agents)
|
|
73
|
+
? body.agents.map((agent) => String(agent).trim()).filter(Boolean)
|
|
74
|
+
: undefined;
|
|
75
|
+
const applySnapshotEval = body.applySnapshotEval !== false;
|
|
76
|
+
if (!configPathRaw) {
|
|
77
|
+
asJson(res, 400, { error: 'configPath is required' });
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
if (Number.isNaN(runsPerScenario) || runsPerScenario <= 0) {
|
|
81
|
+
asJson(res, 400, { error: 'runsPerScenario must be a positive number' });
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
const configPath = isAbsolute(configPathRaw)
|
|
85
|
+
? ensureInsideRoot(settings.evalsDir, configPathRaw)
|
|
86
|
+
: ensureInsideRoot(settings.evalsDir, join(settings.evalsDir, configPathRaw));
|
|
87
|
+
if (!existsSync(configPath)) {
|
|
88
|
+
asJson(res, 404, { error: `Config not found: ${configPath}` });
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
const jobId = `${Date.now()}`;
|
|
92
|
+
const job = {
|
|
93
|
+
id: jobId,
|
|
94
|
+
status: 'running',
|
|
95
|
+
events: [],
|
|
96
|
+
clients: new Set(),
|
|
97
|
+
abortController: new AbortController()
|
|
98
|
+
};
|
|
99
|
+
jobs.set(jobId, job);
|
|
100
|
+
activeJobState.set(jobId);
|
|
101
|
+
addJobEvent(job, {
|
|
102
|
+
type: 'started',
|
|
103
|
+
ts: new Date().toISOString(),
|
|
104
|
+
payload: {
|
|
105
|
+
configPath,
|
|
106
|
+
runsPerScenario,
|
|
107
|
+
scenarioId: scenarioId ?? null,
|
|
108
|
+
scenarioIds: scenarioIds ?? null,
|
|
109
|
+
agents: requestedAgents ?? null
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
void (async () => {
|
|
113
|
+
try {
|
|
114
|
+
addJobEvent(job, {
|
|
115
|
+
type: 'log',
|
|
116
|
+
ts: new Date().toISOString(),
|
|
117
|
+
payload: { message: `Loading MCP Evaluation config: ${configPath}` }
|
|
118
|
+
});
|
|
119
|
+
const loaded = loadConfig(configPath, { bundleRoot: settings.librariesDir });
|
|
120
|
+
addJobEvent(job, {
|
|
121
|
+
type: 'log',
|
|
122
|
+
ts: new Date().toISOString(),
|
|
123
|
+
payload: {
|
|
124
|
+
message: `Loaded config (${loaded.config.scenarios.length} scenario(s), ${Object.keys(loaded.config.agents ?? {}).length} agent(s), ${Object.keys(loaded.config.servers ?? {}).length} server(s))`
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
for (const warning of loaded.warnings ?? []) {
|
|
128
|
+
addJobEvent(job, {
|
|
129
|
+
type: 'log',
|
|
130
|
+
ts: new Date().toISOString(),
|
|
131
|
+
payload: { message: warning }
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
addJobEvent(job, {
|
|
135
|
+
type: 'log',
|
|
136
|
+
ts: new Date().toISOString(),
|
|
137
|
+
payload: {
|
|
138
|
+
message: scenarioIds && scenarioIds.length > 0
|
|
139
|
+
? `Selecting requested scenarios: ${scenarioIds.join(', ')}`
|
|
140
|
+
: scenarioId
|
|
141
|
+
? `Selecting requested scenario: ${scenarioId}`
|
|
142
|
+
: 'Using all scenarios from config'
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
const selectedBaseScenarios = selectScenarioIds(loaded.config, scenarioIds && scenarioIds.length > 0
|
|
146
|
+
? scenarioIds
|
|
147
|
+
: scenarioId
|
|
148
|
+
? [scenarioId]
|
|
149
|
+
: undefined);
|
|
150
|
+
addJobEvent(job, {
|
|
151
|
+
type: 'log',
|
|
152
|
+
ts: new Date().toISOString(),
|
|
153
|
+
payload: {
|
|
154
|
+
message: `Selected ${selectedBaseScenarios.scenarios.length} base scenario(s)`
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
const resolvedAgents = resolveRunSelectedAgents(selectedBaseScenarios, requestedAgents);
|
|
158
|
+
const resolvedAgentList = Array.isArray(resolvedAgents) ? resolvedAgents : [];
|
|
159
|
+
addJobEvent(job, {
|
|
160
|
+
type: 'log',
|
|
161
|
+
ts: new Date().toISOString(),
|
|
162
|
+
payload: {
|
|
163
|
+
message: requestedAgents && requestedAgents.length > 0
|
|
164
|
+
? `Using requested agents: ${resolvedAgentList.join(', ')}`
|
|
165
|
+
: `Using resolved default agents: ${resolvedAgentList.join(', ')}`
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
const expandedConfig = expandConfigForAgents(selectedBaseScenarios, resolvedAgents);
|
|
169
|
+
addJobEvent(job, {
|
|
170
|
+
type: 'log',
|
|
171
|
+
ts: new Date().toISOString(),
|
|
172
|
+
payload: {
|
|
173
|
+
message: `Expanded to ${expandedConfig.scenarios.length} executable scenario run(s) across selected agents`
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
const cwdBefore = process.cwd();
|
|
177
|
+
process.chdir(settings.workspaceRoot);
|
|
178
|
+
try {
|
|
179
|
+
addJobEvent(job, {
|
|
180
|
+
type: 'log',
|
|
181
|
+
ts: new Date().toISOString(),
|
|
182
|
+
payload: {
|
|
183
|
+
message: `Running evaluation (${runsPerScenario} run(s) per scenario) ...`
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
const { runDir, results } = await runAll(expandedConfig, {
|
|
187
|
+
runsPerScenario,
|
|
188
|
+
scenarioId,
|
|
189
|
+
configHash: loaded.hash,
|
|
190
|
+
cliVersion: pkgVersion,
|
|
191
|
+
runsDir: settings.runsDir,
|
|
192
|
+
signal: job.abortController.signal,
|
|
193
|
+
onProgress: async (event) => {
|
|
194
|
+
const message = formatRunProgressMessage(event);
|
|
195
|
+
if (!message)
|
|
196
|
+
return;
|
|
197
|
+
addJobEvent(job, {
|
|
198
|
+
type: 'log',
|
|
199
|
+
ts: new Date().toISOString(),
|
|
200
|
+
payload: { message }
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
addJobEvent(job, {
|
|
205
|
+
type: 'log',
|
|
206
|
+
ts: new Date().toISOString(),
|
|
207
|
+
payload: {
|
|
208
|
+
message: `Evaluation execution finished (run id: ${results.metadata.run_id})`
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
if (applySnapshotEval && expandedConfig.snapshot_eval?.enabled) {
|
|
212
|
+
addJobEvent(job, {
|
|
213
|
+
type: 'log',
|
|
214
|
+
ts: new Date().toISOString(),
|
|
215
|
+
payload: { message: 'Applying snapshot evaluation policy ...' }
|
|
216
|
+
});
|
|
217
|
+
const policy = expandedConfig.snapshot_eval;
|
|
218
|
+
const enabledScenarioIds = new Set(selectedBaseScenarios.scenarios
|
|
219
|
+
.filter((scenario) => scenario.snapshot_eval?.enabled !== false)
|
|
220
|
+
.map((scenario) => scenario.id));
|
|
221
|
+
const scenarioBaselineMap = new Map();
|
|
222
|
+
for (const scenario of selectedBaseScenarios.scenarios) {
|
|
223
|
+
if (scenario.snapshot_eval?.enabled === false)
|
|
224
|
+
continue;
|
|
225
|
+
const baselineId = scenario.snapshot_eval?.baseline_snapshot_id ?? policy.baseline_snapshot_id;
|
|
226
|
+
if (baselineId)
|
|
227
|
+
scenarioBaselineMap.set(scenario.id, baselineId);
|
|
228
|
+
}
|
|
229
|
+
const scenariosWithoutBaseline = selectedBaseScenarios.scenarios
|
|
230
|
+
.filter((scenario) => scenario.snapshot_eval?.enabled !== false)
|
|
231
|
+
.filter((scenario) => !(scenario.snapshot_eval?.baseline_snapshot_id ?? policy.baseline_snapshot_id))
|
|
232
|
+
.map((scenario) => scenario.id);
|
|
233
|
+
if (scenariosWithoutBaseline.length > 0) {
|
|
234
|
+
addJobEvent(job, {
|
|
235
|
+
type: 'log',
|
|
236
|
+
ts: new Date().toISOString(),
|
|
237
|
+
payload: {
|
|
238
|
+
message: `Snapshot eval enabled but no baseline configured for scenarios: ${scenariosWithoutBaseline.join(', ')}`
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
const comparisons = [];
|
|
243
|
+
const scenarioIdsByBaseline = new Map();
|
|
244
|
+
for (const [scenarioIdItem, baselineId] of scenarioBaselineMap) {
|
|
245
|
+
const list = scenarioIdsByBaseline.get(baselineId) ?? [];
|
|
246
|
+
list.push(scenarioIdItem);
|
|
247
|
+
scenarioIdsByBaseline.set(baselineId, list);
|
|
248
|
+
}
|
|
249
|
+
for (const [baselineId, scenarioIdsForBaseline] of scenarioIdsByBaseline) {
|
|
250
|
+
addJobEvent(job, {
|
|
251
|
+
type: 'log',
|
|
252
|
+
ts: new Date().toISOString(),
|
|
253
|
+
payload: {
|
|
254
|
+
message: `Comparing ${scenarioIdsForBaseline.length} scenario(s) to snapshot baseline '${baselineId}'`
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
const snapshot = loadSnapshot(baselineId, settings.snapshotsDir);
|
|
258
|
+
const fullComparison = compareRunToSnapshot(results, snapshot);
|
|
259
|
+
comparisons.push({
|
|
260
|
+
...fullComparison,
|
|
261
|
+
scenario_results: fullComparison.scenario_results.filter((row) => scenarioIdsForBaseline.includes(row.scenario_id))
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
if (comparisons.length > 0) {
|
|
265
|
+
applySnapshotPolicyToRunResult({ results, comparisons, policy, enabledScenarioIds });
|
|
266
|
+
addJobEvent(job, {
|
|
267
|
+
type: 'log',
|
|
268
|
+
ts: new Date().toISOString(),
|
|
269
|
+
payload: {
|
|
270
|
+
message: `Snapshot evaluation applied (${comparisons.length} baseline comparison group(s))`
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
else {
|
|
275
|
+
addJobEvent(job, {
|
|
276
|
+
type: 'log',
|
|
277
|
+
ts: new Date().toISOString(),
|
|
278
|
+
payload: {
|
|
279
|
+
message: 'Snapshot evaluation enabled, but no baseline comparisons were applied'
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
else if (applySnapshotEval) {
|
|
285
|
+
addJobEvent(job, {
|
|
286
|
+
type: 'log',
|
|
287
|
+
ts: new Date().toISOString(),
|
|
288
|
+
payload: {
|
|
289
|
+
message: 'Snapshot evaluation requested, but config snapshot evaluation is disabled'
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
addJobEvent(job, {
|
|
295
|
+
type: 'log',
|
|
296
|
+
ts: new Date().toISOString(),
|
|
297
|
+
payload: {
|
|
298
|
+
message: 'Snapshot evaluation skipped for this run (disabled in run request)'
|
|
299
|
+
}
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
addJobEvent(job, {
|
|
303
|
+
type: 'log',
|
|
304
|
+
ts: new Date().toISOString(),
|
|
305
|
+
payload: { message: `Writing results to ${runDir}` }
|
|
306
|
+
});
|
|
307
|
+
writeFileSync(join(runDir, 'results.json'), `${JSON.stringify(results, null, 2)}\n`, 'utf8');
|
|
308
|
+
writeFileSync(join(runDir, 'report.html'), renderReport(results), 'utf8');
|
|
309
|
+
addJobEvent(job, {
|
|
310
|
+
type: 'log',
|
|
311
|
+
ts: new Date().toISOString(),
|
|
312
|
+
payload: {
|
|
313
|
+
message: `Run finished: ${results.summary.total_runs} run(s), pass rate ${Math.round(results.summary.pass_rate * 100)}%`
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
addJobEvent(job, {
|
|
317
|
+
type: 'completed',
|
|
318
|
+
ts: new Date().toISOString(),
|
|
319
|
+
payload: {
|
|
320
|
+
runId: results.metadata.run_id,
|
|
321
|
+
runDir,
|
|
322
|
+
summary: results.summary,
|
|
323
|
+
snapshotEval: results.metadata.snapshot_eval ?? null
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
job.status = 'completed';
|
|
327
|
+
}
|
|
328
|
+
finally {
|
|
329
|
+
process.chdir(cwdBefore);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
const aborted = job.abortController.signal.aborted || job.status === 'stopped';
|
|
334
|
+
addJobEvent(job, {
|
|
335
|
+
type: 'error',
|
|
336
|
+
ts: new Date().toISOString(),
|
|
337
|
+
payload: {
|
|
338
|
+
message: aborted
|
|
339
|
+
? 'Run aborted by user'
|
|
340
|
+
: error instanceof Error
|
|
341
|
+
? error.message
|
|
342
|
+
: String(error)
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
job.status = aborted ? 'stopped' : 'error';
|
|
346
|
+
}
|
|
347
|
+
finally {
|
|
348
|
+
activeJobState.set(null);
|
|
349
|
+
for (const client of job.clients)
|
|
350
|
+
client.end();
|
|
351
|
+
job.clients.clear();
|
|
352
|
+
}
|
|
353
|
+
})();
|
|
354
|
+
asJson(res, 202, { jobId });
|
|
355
|
+
return true;
|
|
356
|
+
}
|
|
357
|
+
if (pathname.startsWith('/api/runs/') && pathname.endsWith('/assistant') && method === 'POST') {
|
|
358
|
+
const runId = pathname.split('/')[3];
|
|
359
|
+
const results = getRunResults(runId, settings.runsDir);
|
|
360
|
+
const body = (await parseBody(req));
|
|
361
|
+
const messagesRaw = Array.isArray(body.messages) ? body.messages : [];
|
|
362
|
+
const messages = messagesRaw
|
|
363
|
+
.filter((m) => !!m && typeof m === 'object')
|
|
364
|
+
.map((m) => ({
|
|
365
|
+
role: (m.role === 'assistant' ? 'assistant' : 'user'),
|
|
366
|
+
content: String(m.text ?? '')
|
|
367
|
+
}))
|
|
368
|
+
.filter((m) => m.content.trim().length > 0);
|
|
369
|
+
if (messages.length === 0) {
|
|
370
|
+
asJson(res, 400, { error: 'messages are required' });
|
|
371
|
+
return true;
|
|
372
|
+
}
|
|
373
|
+
const libraries = readLibraries(settings.librariesDir);
|
|
374
|
+
const assistantAgentName = pickDefaultAssistantAgentName({
|
|
375
|
+
settingsDefault: settings.scenarioAssistantAgentName,
|
|
376
|
+
agentNames: Object.keys(libraries.agents)
|
|
377
|
+
});
|
|
378
|
+
if (!assistantAgentName) {
|
|
379
|
+
asJson(res, 400, {
|
|
380
|
+
error: 'No assistant agent available. Add an agent in Libraries > Agents or configure the Scenario Assistant Agent in Settings.'
|
|
381
|
+
});
|
|
382
|
+
return true;
|
|
383
|
+
}
|
|
384
|
+
const agentConfig = resolveAssistantAgentFromLibraries(libraries, assistantAgentName);
|
|
385
|
+
const scenarioSummaries = results.scenarios.slice(0, 30).map((sc) => ({
|
|
386
|
+
scenario_id: sc.scenario_id,
|
|
387
|
+
agent: sc.agent,
|
|
388
|
+
pass_rate: sc.pass_rate,
|
|
389
|
+
runs: sc.runs.map((r) => ({
|
|
390
|
+
run_index: r.run_index,
|
|
391
|
+
pass: r.pass,
|
|
392
|
+
failures: r.failures,
|
|
393
|
+
tool_calls: r.tool_calls,
|
|
394
|
+
final_text_preview: typeof r.final_text === 'string' && r.final_text.length > 600
|
|
395
|
+
? `${r.final_text.slice(0, 600)}…`
|
|
396
|
+
: r.final_text,
|
|
397
|
+
extracted: r.extracted
|
|
398
|
+
}))
|
|
399
|
+
}));
|
|
400
|
+
const system = [
|
|
401
|
+
'You are the MCP Labs Result Assistant.',
|
|
402
|
+
'Help the user understand MCP evaluation run results, failures, tool behavior, and snapshot drift.',
|
|
403
|
+
'Be concise, practical, and explain what happened and what to check next.',
|
|
404
|
+
'If the user asks for fixes, suggest concrete config/scenario/eval adjustments.',
|
|
405
|
+
'Use markdown for readability when helpful.',
|
|
406
|
+
`Run result context: ${JSON.stringify({
|
|
407
|
+
run_id: results.metadata.run_id,
|
|
408
|
+
timestamp: results.metadata.timestamp,
|
|
409
|
+
config_hash: results.metadata.config_hash,
|
|
410
|
+
summary: results.summary,
|
|
411
|
+
snapshot_eval: results.metadata.snapshot_eval ?? null,
|
|
412
|
+
scenarios: scenarioSummaries
|
|
413
|
+
})}`
|
|
414
|
+
].join('\n');
|
|
415
|
+
try {
|
|
416
|
+
const reply = await chatWithAgent({
|
|
417
|
+
agent: agentConfig,
|
|
418
|
+
messages,
|
|
419
|
+
system
|
|
420
|
+
});
|
|
421
|
+
asJson(res, 200, {
|
|
422
|
+
reply: reply.content ?? '',
|
|
423
|
+
assistantAgentName,
|
|
424
|
+
provider: agentConfig.provider,
|
|
425
|
+
model: agentConfig.model
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
catch (error) {
|
|
429
|
+
asJson(res, 500, {
|
|
430
|
+
error: error instanceof Error ? error.message : String(error)
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
return true;
|
|
434
|
+
}
|
|
435
|
+
if (pathname.startsWith('/api/runs/') &&
|
|
436
|
+
pathname.endsWith('/assistant/apply-report') &&
|
|
437
|
+
method === 'POST') {
|
|
438
|
+
const runId = pathname.split('/')[3];
|
|
439
|
+
// Validate run exists before writing a report for it.
|
|
440
|
+
getRunResults(runId, settings.runsDir);
|
|
441
|
+
const body = (await parseBody(req));
|
|
442
|
+
const markdown = String(body.markdown ?? '');
|
|
443
|
+
const outputPath = String(body.outputPath ?? '').trim() || defaultResultAssistantReportPath(runId, new Date());
|
|
444
|
+
const overwrite = Boolean(body.overwrite);
|
|
445
|
+
if (!markdown.trim()) {
|
|
446
|
+
asJson(res, 400, { error: 'markdown is required' });
|
|
447
|
+
return true;
|
|
448
|
+
}
|
|
449
|
+
try {
|
|
450
|
+
const mcp = new McpClientManager();
|
|
451
|
+
const serverName = 'mcplab';
|
|
452
|
+
await mcp.connectAll({
|
|
453
|
+
[serverName]: {
|
|
454
|
+
transport: 'http',
|
|
455
|
+
url: localMcplabMcpUrl()
|
|
456
|
+
}
|
|
457
|
+
});
|
|
458
|
+
const toolResult = await mcp.callTool(serverName, 'mcplab_write_markdown_report', {
|
|
459
|
+
output_path: outputPath,
|
|
460
|
+
markdown,
|
|
461
|
+
overwrite
|
|
462
|
+
});
|
|
463
|
+
const structured = toolResult && typeof toolResult === 'object' && 'structuredContent' in toolResult
|
|
464
|
+
? toolResult.structuredContent
|
|
465
|
+
: undefined;
|
|
466
|
+
asJson(res, 200, {
|
|
467
|
+
ok: true,
|
|
468
|
+
runId,
|
|
469
|
+
outputPath,
|
|
470
|
+
tool: 'mcplab_write_markdown_report',
|
|
471
|
+
result: toolResult,
|
|
472
|
+
path: structured &&
|
|
473
|
+
typeof structured === 'object' &&
|
|
474
|
+
typeof structured.path === 'string'
|
|
475
|
+
? structured.path
|
|
476
|
+
: undefined
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
catch (error) {
|
|
480
|
+
asJson(res, 500, {
|
|
481
|
+
error: error instanceof Error
|
|
482
|
+
? `${error.message}. Ensure the MCPLab MCP server is running and exposes mcplab_write_markdown_report.`
|
|
483
|
+
: String(error)
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
return true;
|
|
487
|
+
}
|
|
488
|
+
if (pathname.startsWith('/api/runs/') && method === 'GET') {
|
|
489
|
+
const runId = pathname.replace('/api/runs/', '');
|
|
490
|
+
asJson(res, 200, { runId, results: getRunResults(runId, settings.runsDir) });
|
|
491
|
+
return true;
|
|
492
|
+
}
|
|
493
|
+
if (pathname.startsWith('/api/runs/') && method === 'DELETE') {
|
|
494
|
+
const runId = pathname.replace('/api/runs/', '');
|
|
495
|
+
if (!runId || runId.includes('/')) {
|
|
496
|
+
asJson(res, 400, { error: 'Invalid run id' });
|
|
497
|
+
return true;
|
|
498
|
+
}
|
|
499
|
+
const runDir = ensureInsideRoot(settings.runsDir, join(settings.runsDir, runId));
|
|
500
|
+
if (!existsSync(runDir)) {
|
|
501
|
+
asJson(res, 404, { error: 'Run not found' });
|
|
502
|
+
return true;
|
|
503
|
+
}
|
|
504
|
+
rmSync(runDir, { recursive: true, force: true });
|
|
505
|
+
asJson(res, 200, { ok: true });
|
|
506
|
+
return true;
|
|
507
|
+
}
|
|
508
|
+
return false;
|
|
509
|
+
}
|
|
510
|
+
function formatRunProgressMessage(event) {
|
|
511
|
+
switch (event.type) {
|
|
512
|
+
case 'run_started':
|
|
513
|
+
return `Run initialized (id: ${event.runId}, ${event.totalScenarioRuns} scenario run(s))`;
|
|
514
|
+
case 'mcp_connect_started':
|
|
515
|
+
return `Connecting to ${event.serverCount} MCP server(s) ...`;
|
|
516
|
+
case 'mcp_connect_finished':
|
|
517
|
+
return `Connected to ${event.serverCount} MCP server(s)`;
|
|
518
|
+
case 'scenario_run_started':
|
|
519
|
+
return `Scenario ${event.scenarioRunIndex}/${event.totalScenarioRuns} started: ${event.scenarioId} [agent=${event.agentName}, run=${event.runIndex + 1}/${event.runsPerScenario}]`;
|
|
520
|
+
case 'scenario_run_finished':
|
|
521
|
+
return `Scenario ${event.scenarioRunIndex}/${event.totalScenarioRuns} finished: ${event.scenarioId} [agent=${event.agentName}] -> ${event.pass ? 'PASS' : 'FAIL'} (${event.toolCallCount} tool call(s))`;
|
|
522
|
+
case 'agent_progress': {
|
|
523
|
+
const p = event.event;
|
|
524
|
+
switch (p.type) {
|
|
525
|
+
case 'llm_request_started':
|
|
526
|
+
return `LLM turn ${p.turn + 1} started for ${p.scenarioId} [${p.agentName}] (${p.provider}/${p.model})`;
|
|
527
|
+
case 'llm_response_received':
|
|
528
|
+
return `LLM turn ${p.turn + 1} response for ${p.scenarioId} [${p.agentName}] (text=${p.hasText ? 'yes' : 'no'}, tool_calls=${p.toolCallCount})`;
|
|
529
|
+
case 'tool_call_started':
|
|
530
|
+
return `Tool call started: ${p.server}.${p.tool} (turn ${p.turn + 1})`;
|
|
531
|
+
case 'tool_call_finished':
|
|
532
|
+
return `Tool call ${p.ok ? 'finished' : 'failed'}: ${p.server}.${p.tool} in ${p.durationMs}ms`;
|
|
533
|
+
case 'final_answer':
|
|
534
|
+
return `Final answer produced for ${p.scenarioId} [${p.agentName}] (text=${p.hasText ? 'yes' : 'no'})`;
|
|
535
|
+
default:
|
|
536
|
+
return null;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
case 'run_finished':
|
|
540
|
+
return `Run finished (id: ${event.runId})`;
|
|
541
|
+
default:
|
|
542
|
+
return null;
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
function localMcplabMcpUrl() {
|
|
546
|
+
const host = process.env.MCP_HOST || '127.0.0.1';
|
|
547
|
+
const port = process.env.MCP_PORT || '3011';
|
|
548
|
+
const path = process.env.MCP_PATH || '/mcp';
|
|
549
|
+
return `http://${host}:${port}${path}`;
|
|
550
|
+
}
|
|
551
|
+
function defaultResultAssistantReportPath(runId, now) {
|
|
552
|
+
const stamp = now.toISOString().replace(/[:]/g, '-').replace(/\..+/, '');
|
|
553
|
+
return `mcplab/reports/result-assistant/${runId}-${stamp}.md`;
|
|
554
|
+
}
|
|
555
|
+
//# sourceMappingURL=runs-routes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runs-routes.js","sourceRoot":"","sources":["../../src/app-server/runs-routes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,MAAM,EAIP,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AA8C1D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAStC;IACC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IACpF,MAAM,EACJ,SAAS,EACT,MAAM,EACN,WAAW,EACX,YAAY,EACZ,gBAAgB,EAChB,QAAQ,EACR,aAAa,EACb,0BAA0B,EAC1B,iBAAiB,EACjB,qBAAqB,EACrB,wBAAwB,EACxB,YAAY,EACZ,oBAAoB,EACpB,8BAA8B,EAC9B,aAAa,EACb,6BAA6B,EAC7B,kCAAkC,EAClC,aAAa,EACb,UAAU,EACX,GAAG,IAAI,CAAC;IAET,IAAI,QAAQ,KAAK,WAAW,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACzF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,0BAA0B,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1F,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC/F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;QACrB,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;QACnD,GAAG,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;QAC3C,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAC1C,IAAI,cAAc,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACpE,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM;YAAE,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,GAAG,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC9F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,GAAG,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC5B,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,KAAK,WAAW,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAClD,IAAI,cAAc,CAAC,GAAG,EAAE,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,+BAA+B,EAAE,KAAK,EAAE,cAAc,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,CAAmB,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QACpD,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzE,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAW,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAC1E,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;YAChD,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAC3E,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,KAAK,KAAK,CAAC;QAE3D,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,2CAA2C,EAAE,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;YACpD,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;QAChF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,qBAAqB,UAAU,EAAE,EAAE,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAW;YAClB,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,IAAI,GAAG,EAAE;YAClB,eAAe,EAAE,IAAI,eAAe,EAAE;SACvC,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACrB,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAE1B,WAAW,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,SAAS;YACf,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,OAAO,EAAE;gBACP,UAAU;gBACV,eAAe;gBACf,UAAU,EAAE,UAAU,IAAI,IAAI;gBAC9B,WAAW,EAAE,WAAW,IAAI,IAAI;gBAChC,MAAM,EAAE,eAAe,IAAI,IAAI;aAChC;SACF,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,KAAK;oBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE,EAAE,OAAO,EAAE,kCAAkC,UAAU,EAAE,EAAE;iBACrE,CAAC,CAAC;gBACH,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC;gBAC7E,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,KAAK;oBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,kBAAkB,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,iBACvD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAC1C,cAAc,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,aAAa;qBAC3E;iBACF,CAAC,CAAC;gBACH,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBAC5C,WAAW,CAAC,GAAG,EAAE;wBACf,IAAI,EAAE,KAAK;wBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBACD,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,KAAK;oBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EACL,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;4BACnC,CAAC,CAAC,kCAAkC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;4BAC5D,CAAC,CAAC,UAAU;gCACZ,CAAC,CAAC,iCAAiC,UAAU,EAAE;gCAC/C,CAAC,CAAC,iCAAiC;qBACxC;iBACF,CAAC,CAAC;gBACH,MAAM,qBAAqB,GAAG,iBAAiB,CAC7C,MAAM,CAAC,MAAM,EACb,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBACnC,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,UAAU;wBACZ,CAAC,CAAC,CAAC,UAAU,CAAC;wBACd,CAAC,CAAC,SAAS,CACd,CAAC;gBACF,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,KAAK;oBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,YAAY,qBAAqB,CAAC,SAAS,CAAC,MAAM,mBAAmB;qBAC/E;iBACF,CAAC,CAAC;gBACH,MAAM,cAAc,GAAG,wBAAwB,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAC;gBACxF,MAAM,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,KAAK;oBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EACL,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;4BAC3C,CAAC,CAAC,2BAA2B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;4BAC3D,CAAC,CAAC,kCAAkC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBACvE;iBACF,CAAC,CAAC;gBACH,MAAM,cAAc,GAAG,qBAAqB,CAAC,qBAAqB,EAAE,cAAc,CAAC,CAAC;gBACpF,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,KAAK;oBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,eAAe,cAAc,CAAC,SAAS,CAAC,MAAM,oDAAoD;qBAC5G;iBACF,CAAC,CAAC;gBACH,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;gBACtC,IAAI,CAAC;oBACH,WAAW,CAAC,GAAG,EAAE;wBACf,IAAI,EAAE,KAAK;wBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,OAAO,EAAE;4BACP,OAAO,EAAE,uBAAuB,eAAe,2BAA2B;yBAC3E;qBACF,CAAC,CAAC;oBACH,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE;wBACvD,eAAe;wBACf,UAAU;wBACV,UAAU,EAAE,MAAM,CAAC,IAAI;wBACvB,UAAU,EAAE,UAAU;wBACtB,OAAO,EAAE,QAAQ,CAAC,OAAO;wBACzB,MAAM,EAAE,GAAG,CAAC,eAAe,CAAC,MAAM;wBAClC,UAAU,EAAE,KAAK,EAAE,KAAuB,EAAE,EAAE;4BAC5C,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;4BAChD,IAAI,CAAC,OAAO;gCAAE,OAAO;4BACrB,WAAW,CAAC,GAAG,EAAE;gCACf,IAAI,EAAE,KAAK;gCACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gCAC5B,OAAO,EAAE,EAAE,OAAO,EAAE;6BACrB,CAAC,CAAC;wBACL,CAAC;qBACF,CAAC,CAAC;oBACH,WAAW,CAAC,GAAG,EAAE;wBACf,IAAI,EAAE,KAAK;wBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,OAAO,EAAE;4BACP,OAAO,EAAE,0CAA0C,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG;yBAC9E;qBACF,CAAC,CAAC;oBACH,IAAI,iBAAiB,IAAI,cAAc,CAAC,aAAa,EAAE,OAAO,EAAE,CAAC;wBAC/D,WAAW,CAAC,GAAG,EAAE;4BACf,IAAI,EAAE,KAAK;4BACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BAC5B,OAAO,EAAE,EAAE,OAAO,EAAE,yCAAyC,EAAE;yBAChE,CAAC,CAAC;wBACH,MAAM,MAAM,GAAG,cAAc,CAAC,aAAa,CAAC;wBAC5C,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,qBAAqB,CAAC,SAAS;6BAC5B,MAAM,CAAC,CAAC,QAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,KAAK,KAAK,CAAC;6BAC/E,GAAG,CAAC,CAAC,QAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAClD,CAAC;wBACF,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;wBACtD,KAAK,MAAM,QAAQ,IAAI,qBAAqB,CAAC,SAAS,EAAE,CAAC;4BACvD,IAAI,QAAQ,CAAC,aAAa,EAAE,OAAO,KAAK,KAAK;gCAAE,SAAS;4BACxD,MAAM,UAAU,GACd,QAAQ,CAAC,aAAa,EAAE,oBAAoB,IAAI,MAAM,CAAC,oBAAoB,CAAC;4BAC9E,IAAI,UAAU;gCAAE,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;wBACnE,CAAC;wBACD,MAAM,wBAAwB,GAAG,qBAAqB,CAAC,SAAS;6BAC7D,MAAM,CAAC,CAAC,QAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,KAAK,KAAK,CAAC;6BAC/E,MAAM,CACL,CAAC,QAAwB,EAAE,EAAE,CAC3B,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,oBAAoB,IAAI,MAAM,CAAC,oBAAoB,CAAC,CACjF;6BACA,GAAG,CAAC,CAAC,QAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;wBAClD,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACxC,WAAW,CAAC,GAAG,EAAE;gCACf,IAAI,EAAE,KAAK;gCACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gCAC5B,OAAO,EAAE;oCACP,OAAO,EAAE,mEAAmE,wBAAwB,CAAC,IAAI,CACvG,IAAI,CACL,EAAE;iCACJ;6BACF,CAAC,CAAC;wBACL,CAAC;wBACD,MAAM,WAAW,GAAwD,EAAE,CAAC;wBAC5E,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAoB,CAAC;wBAC1D,KAAK,MAAM,CAAC,cAAc,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;4BAC/D,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;4BACzD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;4BAC1B,qBAAqB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;wBAC9C,CAAC;wBACD,KAAK,MAAM,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,qBAAqB,EAAE,CAAC;4BACzE,WAAW,CAAC,GAAG,EAAE;gCACf,IAAI,EAAE,KAAK;gCACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gCAC5B,OAAO,EAAE;oCACP,OAAO,EAAE,aAAa,sBAAsB,CAAC,MAAM,sCAAsC,UAAU,GAAG;iCACvG;6BACF,CAAC,CAAC;4BACH,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;4BACjE,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;4BAC/D,WAAW,CAAC,IAAI,CAAC;gCACf,GAAG,cAAc;gCACjB,gBAAgB,EAAE,cAAc,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/D,sBAAsB,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CACjD;6BACF,CAAC,CAAC;wBACL,CAAC;wBACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC3B,8BAA8B,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;4BACrF,WAAW,CAAC,GAAG,EAAE;gCACf,IAAI,EAAE,KAAK;gCACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gCAC5B,OAAO,EAAE;oCACP,OAAO,EAAE,gCAAgC,WAAW,CAAC,MAAM,gCAAgC;iCAC5F;6BACF,CAAC,CAAC;wBACL,CAAC;6BAAM,CAAC;4BACN,WAAW,CAAC,GAAG,EAAE;gCACf,IAAI,EAAE,KAAK;gCACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gCAC5B,OAAO,EAAE;oCACP,OAAO,EAAE,uEAAuE;iCACjF;6BACF,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,iBAAiB,EAAE,CAAC;wBAC7B,WAAW,CAAC,GAAG,EAAE;4BACf,IAAI,EAAE,KAAK;4BACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BAC5B,OAAO,EAAE;gCACP,OAAO,EAAE,2EAA2E;6BACrF;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,WAAW,CAAC,GAAG,EAAE;4BACf,IAAI,EAAE,KAAK;4BACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BAC5B,OAAO,EAAE;gCACP,OAAO,EAAE,oEAAoE;6BAC9E;yBACF,CAAC,CAAC;oBACL,CAAC;oBACD,WAAW,CAAC,GAAG,EAAE;wBACf,IAAI,EAAE,KAAK;wBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,OAAO,EAAE,EAAE,OAAO,EAAE,sBAAsB,MAAM,EAAE,EAAE;qBACrD,CAAC,CAAC;oBACH,aAAa,CACX,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAC5B,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EACvC,MAAM,CACP,CAAC;oBACF,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;oBAC1E,WAAW,CAAC,GAAG,EAAE;wBACf,IAAI,EAAE,KAAK;wBACX,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,OAAO,EAAE;4BACP,OAAO,EAAE,iBAAiB,OAAO,CAAC,OAAO,CAAC,UAAU,sBAAsB,IAAI,CAAC,KAAK,CAClF,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,GAAG,CAChC,GAAG;yBACL;qBACF,CAAC,CAAC;oBACH,WAAW,CAAC,GAAG,EAAE;wBACf,IAAI,EAAE,WAAW;wBACjB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBAC5B,OAAO,EAAE;4BACP,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;4BAC9B,MAAM;4BACN,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,IAAI,IAAI;yBACrD;qBACF,CAAC,CAAC;oBACH,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;gBAC3B,CAAC;wBAAS,CAAC;oBACT,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;gBAC/E,WAAW,CAAC,GAAG,EAAE;oBACf,IAAI,EAAE,OAAO;oBACb,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,OAAO,EAAE;wBACP,OAAO,EAAE,OAAO;4BACd,CAAC,CAAC,qBAAqB;4BACvB,CAAC,CAAC,KAAK,YAAY,KAAK;gCACxB,CAAC,CAAC,KAAK,CAAC,OAAO;gCACf,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;qBAClB;iBACF,CAAC,CAAC;gBACH,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YAC7C,CAAC;oBAAS,CAAC;gBACT,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzB,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO;oBAAE,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC/C,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAC9F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,CAEjC,CAAC;QACF,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAiB,WAAW;aACvC,MAAM,CAAC,CAAC,CAAC,EAA2C,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;aACpF,GAAG,CACF,CAAC,CAAC,EAAc,EAAE,CAAC,CAAC;YAClB,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAyB;YAC7E,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;SAC9B,CAAC,CACH;aACA,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;YACvD,eAAe,EAAE,QAAQ,CAAC,0BAA0B;YACpD,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;gBACf,KAAK,EACH,yHAAyH;aAC5H,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,WAAW,GAAG,kCAAkC,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QAEtF,MAAM,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACpE,WAAW,EAAE,EAAE,CAAC,WAAW;YAC3B,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,kBAAkB,EAChB,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG;oBAC3D,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG;oBAClC,CAAC,CAAC,CAAC,CAAC,UAAU;gBAClB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC;QAEJ,MAAM,MAAM,GAAG;YACb,wCAAwC;YACxC,mGAAmG;YACnG,0EAA0E;YAC1E,gFAAgF;YAChF,4CAA4C;YAC5C,uBAAuB,IAAI,CAAC,SAAS,CAAC;gBACpC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAC/B,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;gBACrC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW;gBACzC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,IAAI,IAAI;gBACrD,SAAS,EAAE,iBAAiB;aAC7B,CAAC,EAAE;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC;gBAChC,KAAK,EAAE,WAAW;gBAClB,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;YACH,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;gBACf,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE;gBAC1B,kBAAkB;gBAClB,QAAQ,EAAE,WAAW,CAAC,QAAQ;gBAC9B,KAAK,EAAE,WAAW,CAAC,KAAK;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;gBACf,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IACE,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC;QACjC,QAAQ,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAC5C,MAAM,KAAK,MAAM,EACjB,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,sDAAsD;QACtD,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,CAIjC,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC7C,MAAM,UAAU,GACd,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,gCAAgC,CAAC,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;QAC9F,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE,CAAC,CAAC;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,QAAQ,CAAC;YAC5B,MAAM,GAAG,CAAC,UAAU,CAAC;gBACnB,CAAC,UAAU,CAAC,EAAE;oBACZ,SAAS,EAAE,MAAM;oBACjB,GAAG,EAAE,iBAAiB,EAAE;iBACzB;aACF,CAAC,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,8BAA8B,EAAE;gBAChF,WAAW,EAAE,UAAU;gBACvB,QAAQ;gBACR,SAAS;aACV,CAAC,CAAC;YACH,MAAM,UAAU,GACd,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,mBAAmB,IAAK,UAAkB;gBACxF,CAAC,CAAE,UAAkB,CAAC,iBAAiB;gBACvC,CAAC,CAAC,SAAS,CAAC;YAChB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;gBACf,EAAE,EAAE,IAAI;gBACR,KAAK;gBACL,UAAU;gBACV,IAAI,EAAE,8BAA8B;gBACpC,MAAM,EAAE,UAAU;gBAClB,IAAI,EACF,UAAU;oBACV,OAAO,UAAU,KAAK,QAAQ;oBAC9B,OAAQ,UAAkB,CAAC,IAAI,KAAK,QAAQ;oBAC1C,CAAC,CAAE,UAAkB,CAAC,IAAI;oBAC1B,CAAC,CAAC,SAAS;aAChB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE;gBACf,KAAK,EACH,KAAK,YAAY,KAAK;oBACpB,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,qFAAqF;oBACvG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aACpB,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAuB;IACvD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,aAAa;YAChB,OAAO,wBAAwB,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,iBAAiB,mBAAmB,CAAC;QAC5F,KAAK,qBAAqB;YACxB,OAAO,iBAAiB,KAAK,CAAC,WAAW,oBAAoB,CAAC;QAChE,KAAK,sBAAsB;YACzB,OAAO,gBAAgB,KAAK,CAAC,WAAW,gBAAgB,CAAC;QAC3D,KAAK,sBAAsB;YACzB,OAAO,YAAY,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,iBAAiB,aAClE,KAAK,CAAC,UACR,WAAW,KAAK,CAAC,SAAS,SAAS,KAAK,CAAC,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC;QACpF,KAAK,uBAAuB;YAC1B,OAAO,YAAY,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,iBAAiB,cAClE,KAAK,CAAC,UACR,WAAW,KAAK,CAAC,SAAS,QAAQ,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAC5D,KAAK,CAAC,aACR,gBAAgB,CAAC;QACnB,KAAK,gBAAgB,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC;YACtB,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;gBACf,KAAK,qBAAqB;oBACxB,OAAO,YAAY,CAAC,CAAC,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,SAAS,MACvE,CAAC,CAAC,QACJ,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC;gBACjB,KAAK,uBAAuB;oBAC1B,OAAO,YAAY,CAAC,CAAC,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,SAAS,WACxE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IACtB,gBAAgB,CAAC,CAAC,aAAa,GAAG,CAAC;gBACrC,KAAK,mBAAmB;oBACtB,OAAO,sBAAsB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC;gBACzE,KAAK,oBAAoB;oBACvB,OAAO,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,OACrE,CAAC,CAAC,UACJ,IAAI,CAAC;gBACP,KAAK,cAAc;oBACjB,OAAO,6BAA6B,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,SAAS,WAC9D,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IACtB,GAAG,CAAC;gBACN;oBACE,OAAO,IAAI,CAAC;YAChB,CAAC;QACH,CAAC;QACD,KAAK,cAAc;YACjB,OAAO,qBAAqB,KAAK,CAAC,KAAK,GAAG,CAAC;QAC7C;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,WAAW,CAAC;IACjD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC;IAC5C,OAAO,UAAU,IAAI,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,gCAAgC,CAAC,KAAa,EAAE,GAAS;IAChE,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzE,OAAO,mCAAmC,KAAK,IAAI,KAAK,KAAK,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EvalConfig, ResultsJson, ScenarioRunTraceRecord } from '@inspectr/mcplab-core';
|
|
2
|
+
export interface RunSummary {
|
|
3
|
+
runId: string;
|
|
4
|
+
path: string;
|
|
5
|
+
timestamp: string;
|
|
6
|
+
configHash: string;
|
|
7
|
+
totalScenarios: number;
|
|
8
|
+
totalRuns: number;
|
|
9
|
+
passRate: number;
|
|
10
|
+
avgToolCalls: number;
|
|
11
|
+
avgLatencyMs: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function listRuns(runsDir: string): RunSummary[];
|
|
14
|
+
export declare function getRunResults(runId: string, runsDir: string): ResultsJson;
|
|
15
|
+
export declare function selectScenarioIds(
|
|
16
|
+
config: EvalConfig,
|
|
17
|
+
requestedScenarioIds?: string[]
|
|
18
|
+
): EvalConfig;
|
|
19
|
+
export declare function getScenarioRunTraceRecords(
|
|
20
|
+
runId: string,
|
|
21
|
+
runsDir: string
|
|
22
|
+
): ScenarioRunTraceRecord[];
|
|
23
|
+
//# sourceMappingURL=runs-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runs-store.d.ts","sourceRoot":"","sources":["../../src/app-server/runs-store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAG7F,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE,CA6BtD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,WAAW,CAKzE;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE,MAAM,EAAE,GAAG,UAAU,CAgBjG;AAED,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,sBAAsB,EAAE,CA0B1B"}
|