@inspectr/mcplab 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +762 -0
  2. package/dist/app/android-chrome-192x192.png +0 -0
  3. package/dist/app/android-chrome-512x512.png +0 -0
  4. package/dist/app/apple-touch-icon.png +0 -0
  5. package/dist/app/assets/index-DT-Z4AVG.js +249 -0
  6. package/dist/app/assets/index-EP4WAY8u.css +1 -0
  7. package/dist/app/favicon-16x16.png +0 -0
  8. package/dist/app/favicon-32x32.png +0 -0
  9. package/dist/app/favicon.svg +13 -0
  10. package/dist/app/index.html +26 -0
  11. package/dist/app/inspectr_logo_color.svg +9 -0
  12. package/dist/app/mcp.png +0 -0
  13. package/dist/app/mcp.svg +1 -0
  14. package/dist/app/robots.txt +14 -0
  15. package/dist/app/site.webmanifest +1 -0
  16. package/dist/app-server/app-context.d.ts +164 -0
  17. package/dist/app-server/app-context.d.ts.map +1 -0
  18. package/dist/app-server/app-context.js +2 -0
  19. package/dist/app-server/app-context.js.map +1 -0
  20. package/dist/app-server/assistant-common.d.ts +41 -0
  21. package/dist/app-server/assistant-common.d.ts.map +1 -0
  22. package/dist/app-server/assistant-common.js +104 -0
  23. package/dist/app-server/assistant-common.js.map +1 -0
  24. package/dist/app-server/config-store.d.ts +15 -0
  25. package/dist/app-server/config-store.d.ts.map +1 -0
  26. package/dist/app-server/config-store.js +67 -0
  27. package/dist/app-server/config-store.js.map +1 -0
  28. package/dist/app-server/dev-mcp.d.ts +6 -0
  29. package/dist/app-server/dev-mcp.d.ts.map +1 -0
  30. package/dist/app-server/dev-mcp.js +71 -0
  31. package/dist/app-server/dev-mcp.js.map +1 -0
  32. package/dist/app-server/evals-routes.d.ts +22 -0
  33. package/dist/app-server/evals-routes.d.ts.map +1 -0
  34. package/dist/app-server/evals-routes.js +135 -0
  35. package/dist/app-server/evals-routes.js.map +1 -0
  36. package/dist/app-server/http.d.ts +5 -0
  37. package/dist/app-server/http.d.ts.map +1 -0
  38. package/dist/app-server/http.js +31 -0
  39. package/dist/app-server/http.js.map +1 -0
  40. package/dist/app-server/index.d.ts +3 -0
  41. package/dist/app-server/index.d.ts.map +1 -0
  42. package/dist/app-server/index.js +2 -0
  43. package/dist/app-server/index.js.map +1 -0
  44. package/dist/app-server/jobs.d.ts +15 -0
  45. package/dist/app-server/jobs.d.ts.map +1 -0
  46. package/dist/app-server/jobs.js +11 -0
  47. package/dist/app-server/jobs.js.map +1 -0
  48. package/dist/app-server/libraries-store.d.ts +15 -0
  49. package/dist/app-server/libraries-store.d.ts.map +1 -0
  50. package/dist/app-server/libraries-store.js +61 -0
  51. package/dist/app-server/libraries-store.js.map +1 -0
  52. package/dist/app-server/markdown-reports.d.ts +12 -0
  53. package/dist/app-server/markdown-reports.d.ts.map +1 -0
  54. package/dist/app-server/markdown-reports.js +145 -0
  55. package/dist/app-server/markdown-reports.js.map +1 -0
  56. package/dist/app-server/oauth-debugger-domain.d.ts +230 -0
  57. package/dist/app-server/oauth-debugger-domain.d.ts.map +1 -0
  58. package/dist/app-server/oauth-debugger-domain.js +1098 -0
  59. package/dist/app-server/oauth-debugger-domain.js.map +1 -0
  60. package/dist/app-server/oauth-debugger.d.ts +20 -0
  61. package/dist/app-server/oauth-debugger.d.ts.map +1 -0
  62. package/dist/app-server/oauth-debugger.js +193 -0
  63. package/dist/app-server/oauth-debugger.js.map +1 -0
  64. package/dist/app-server/provider-models.d.ts +8 -0
  65. package/dist/app-server/provider-models.d.ts.map +1 -0
  66. package/dist/app-server/provider-models.js +60 -0
  67. package/dist/app-server/provider-models.js.map +1 -0
  68. package/dist/app-server/result-assistant-domain.d.ts +87 -0
  69. package/dist/app-server/result-assistant-domain.d.ts.map +1 -0
  70. package/dist/app-server/result-assistant-domain.js +212 -0
  71. package/dist/app-server/result-assistant-domain.js.map +1 -0
  72. package/dist/app-server/result-assistant.d.ts +22 -0
  73. package/dist/app-server/result-assistant.d.ts.map +1 -0
  74. package/dist/app-server/result-assistant.js +328 -0
  75. package/dist/app-server/result-assistant.js.map +1 -0
  76. package/dist/app-server/router.d.ts +4 -0
  77. package/dist/app-server/router.d.ts.map +1 -0
  78. package/dist/app-server/router.js +374 -0
  79. package/dist/app-server/router.js.map +1 -0
  80. package/dist/app-server/runs-routes.d.ts +44 -0
  81. package/dist/app-server/runs-routes.d.ts.map +1 -0
  82. package/dist/app-server/runs-routes.js +555 -0
  83. package/dist/app-server/runs-routes.js.map +1 -0
  84. package/dist/app-server/runs-store.d.ts +23 -0
  85. package/dist/app-server/runs-store.d.ts.map +1 -0
  86. package/dist/app-server/runs-store.js +84 -0
  87. package/dist/app-server/runs-store.js.map +1 -0
  88. package/dist/app-server/scenario-assistant-domain.d.ts +162 -0
  89. package/dist/app-server/scenario-assistant-domain.d.ts.map +1 -0
  90. package/dist/app-server/scenario-assistant-domain.js +269 -0
  91. package/dist/app-server/scenario-assistant-domain.js.map +1 -0
  92. package/dist/app-server/scenario-assistant.d.ts +29 -0
  93. package/dist/app-server/scenario-assistant.d.ts.map +1 -0
  94. package/dist/app-server/scenario-assistant.js +246 -0
  95. package/dist/app-server/scenario-assistant.js.map +1 -0
  96. package/dist/app-server/settings-store.d.ts +4 -0
  97. package/dist/app-server/settings-store.d.ts.map +1 -0
  98. package/dist/app-server/settings-store.js +32 -0
  99. package/dist/app-server/settings-store.js.map +1 -0
  100. package/dist/app-server/snapshots-routes.d.ts +24 -0
  101. package/dist/app-server/snapshots-routes.d.ts.map +1 -0
  102. package/dist/app-server/snapshots-routes.js +82 -0
  103. package/dist/app-server/snapshots-routes.js.map +1 -0
  104. package/dist/app-server/static-serving.d.ts +17 -0
  105. package/dist/app-server/static-serving.d.ts.map +1 -0
  106. package/dist/app-server/static-serving.js +64 -0
  107. package/dist/app-server/static-serving.js.map +1 -0
  108. package/dist/app-server/store-utils.d.ts +5 -0
  109. package/dist/app-server/store-utils.d.ts.map +1 -0
  110. package/dist/app-server/store-utils.js +26 -0
  111. package/dist/app-server/store-utils.js.map +1 -0
  112. package/dist/app-server/tool-analysis-domain.d.ts +146 -0
  113. package/dist/app-server/tool-analysis-domain.d.ts.map +1 -0
  114. package/dist/app-server/tool-analysis-domain.js +556 -0
  115. package/dist/app-server/tool-analysis-domain.js.map +1 -0
  116. package/dist/app-server/tool-analysis-storage.d.ts +41 -0
  117. package/dist/app-server/tool-analysis-storage.d.ts.map +1 -0
  118. package/dist/app-server/tool-analysis-storage.js +110 -0
  119. package/dist/app-server/tool-analysis-storage.js.map +1 -0
  120. package/dist/app-server/tool-analysis.d.ts +22 -0
  121. package/dist/app-server/tool-analysis.d.ts.map +1 -0
  122. package/dist/app-server/tool-analysis.js +271 -0
  123. package/dist/app-server/tool-analysis.js.map +1 -0
  124. package/dist/app-server/types.d.ts +28 -0
  125. package/dist/app-server/types.d.ts.map +1 -0
  126. package/dist/app-server/types.js +2 -0
  127. package/dist/app-server/types.js.map +1 -0
  128. package/dist/cli.d.ts +3 -0
  129. package/dist/cli.d.ts.map +1 -0
  130. package/dist/cli.js +544 -0
  131. package/dist/cli.js.map +1 -0
  132. package/dist/snapshot.d.ts +80 -0
  133. package/dist/snapshot.d.ts.map +1 -0
  134. package/dist/snapshot.js +401 -0
  135. package/dist/snapshot.js.map +1 -0
  136. package/package.json +55 -0
package/dist/cli.js ADDED
@@ -0,0 +1,544 @@
1
+ #!/usr/bin/env node
2
+ import 'dotenv/config';
3
+ import { Command } from 'commander';
4
+ import kleur from 'kleur';
5
+ import { readFileSync, readdirSync, writeFileSync } from 'node:fs';
6
+ import { join, resolve } from 'node:path';
7
+ import { loadConfig, selectScenarios, runAll, expandConfigForAgents } from '@inspectr/mcplab-core';
8
+ import { renderReport } from '@inspectr/mcplab-reporting';
9
+ import { execSync } from 'node:child_process';
10
+ import chokidar from 'chokidar';
11
+ import { stringify as stringifyYaml } from 'yaml';
12
+ import { startAppServer } from './app-server/index.js';
13
+ import { applySnapshotPolicyToRunResult, buildSnapshotFromRun, compareRunToSnapshot, formatSnapshotComparisonTable, listSnapshots, loadSnapshot, saveSnapshot } from './snapshot.js';
14
+ const pkgVersion = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
15
+ ?.version;
16
+ const program = new Command();
17
+ program
18
+ .name('mcplab')
19
+ .description('Laboratory for testing Model Context Protocol servers')
20
+ .version(pkgVersion);
21
+ program
22
+ .command('run')
23
+ .description('Run evaluation scenarios')
24
+ .requiredOption('-c, --config <path>', 'Path to eval.yaml')
25
+ .option('-s, --scenario <id>', 'Run a single scenario')
26
+ .option('-n, --runs <count>', 'Variance runs', '1')
27
+ .option('--agents <agents>', 'Comma-separated list of agents to test (runs each scenario with each agent)')
28
+ .option('--snapshot-eval', 'Apply snapshot eval policy configured in the config')
29
+ .option('--compare-snapshot <snapshotId>', 'Compare completed run against snapshot id')
30
+ .option('--runs-dir <path>', 'Directory for run artifacts', 'mcplab/results/evaluation-runs')
31
+ .option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
32
+ .action(async (options) => {
33
+ try {
34
+ let { config, hash, warnings } = loadConfig(resolve(options.config));
35
+ for (const warning of warnings) {
36
+ console.log(kleur.yellow(`⚠ ${warning}`));
37
+ }
38
+ const requestedAgents = options.agents
39
+ ? options.agents
40
+ .split(',')
41
+ .map((a) => a.trim())
42
+ .filter(Boolean)
43
+ : undefined;
44
+ const beforeExpandCount = config.scenarios.length;
45
+ const effectiveAgents = requestedAgents ?? config.run_defaults?.selected_agents;
46
+ const expanded = expandConfigForAgents(config, effectiveAgents);
47
+ if (expanded.scenarios.length !== beforeExpandCount || effectiveAgents?.length) {
48
+ const agentCount = effectiveAgents?.length ?? Object.keys(config.agents).length;
49
+ console.log(kleur.cyan(`📊 Testing ${beforeExpandCount} scenarios × ${agentCount} selected agents = ${expanded.scenarios.length} total tests`));
50
+ }
51
+ const selected = selectScenarios(expanded, options.scenario);
52
+ const runsPerScenario = Number(options.runs);
53
+ if (Number.isNaN(runsPerScenario) || runsPerScenario <= 0) {
54
+ throw new Error('Runs must be a positive number');
55
+ }
56
+ const { runDir, results } = await runAll(selected, {
57
+ runsPerScenario,
58
+ scenarioId: options.scenario,
59
+ configHash: hash,
60
+ gitCommit: getGitCommit(),
61
+ cliVersion: pkgVersion,
62
+ runsDir: String(options.runsDir)
63
+ });
64
+ let shouldFailOnDrift = false;
65
+ const useSnapshotEval = Boolean(options.snapshotEval) || Boolean(config.snapshot_eval?.enabled);
66
+ if (useSnapshotEval) {
67
+ const policy = config.snapshot_eval;
68
+ if (!policy?.baseline_snapshot_id) {
69
+ console.log(kleur.yellow('⚠ Snapshot eval enabled but no baseline snapshot is configured.'));
70
+ }
71
+ else {
72
+ const snapshot = loadSnapshot(String(policy.baseline_snapshot_id), resolve(options.snapshotsDir));
73
+ const comparison = compareRunToSnapshot(results, snapshot);
74
+ const enabledScenarioIds = new Set(selected.scenarios
75
+ .filter((scenario) => scenario.snapshot_eval?.enabled !== false)
76
+ .map((scenario) => scenario.id));
77
+ const applied = applySnapshotPolicyToRunResult({
78
+ results,
79
+ comparisons: [comparison],
80
+ policy,
81
+ enabledScenarioIds
82
+ });
83
+ console.log('');
84
+ console.log(kleur.cyan('📸 Snapshot Eval Policy'));
85
+ console.log(`${applied.mode} · baseline=${applied.baseline_snapshot_id} · overall=${applied.overall_score} · status=${applied.status}`);
86
+ if (applied.impacted_scenarios.length > 0) {
87
+ console.log(kleur.yellow(`Impacted scenarios: ${applied.impacted_scenarios.join(', ')}`));
88
+ }
89
+ console.log(formatSnapshotComparisonTable(comparison));
90
+ shouldFailOnDrift =
91
+ policy.mode === 'fail_on_drift' && applied.impacted_scenarios.length > 0;
92
+ }
93
+ }
94
+ const reportPath = join(runDir, 'report.html');
95
+ const resultsPath = join(runDir, 'results.json');
96
+ writeFileSync(resultsPath, `${JSON.stringify(results, null, 2)}\n`, 'utf8');
97
+ writeFileSync(reportPath, renderReport(results), 'utf8');
98
+ console.log(kleur.green(`Run completed: ${runDir}`));
99
+ if (options.compareSnapshot) {
100
+ const snapshot = loadSnapshot(String(options.compareSnapshot), resolve(options.snapshotsDir));
101
+ const comparison = compareRunToSnapshot(results, snapshot);
102
+ console.log('');
103
+ console.log(kleur.cyan('📸 Snapshot Comparison'));
104
+ console.log(formatSnapshotComparisonTable(comparison));
105
+ }
106
+ // If multi-agent test, show comparison
107
+ if (options.agents) {
108
+ console.log(kleur.cyan(`\n📈 Run comparison script:`));
109
+ console.log(kleur.gray(` node scripts/compare-llm-results.mjs ${join(runDir, 'results.json')}`));
110
+ }
111
+ if (shouldFailOnDrift) {
112
+ console.error(kleur.red('Snapshot eval drift detected in fail_on_drift mode.'));
113
+ process.exit(2);
114
+ }
115
+ }
116
+ catch (err) {
117
+ const message = err?.message ?? String(err);
118
+ const hint = message.includes('fetch failed')
119
+ ? ' Hint: verify the MCP server is running, the SSE URL is correct, and any bearer token env var is set.'
120
+ : '';
121
+ console.error(kleur.red(`Error: ${message}${hint}`));
122
+ process.exit(1);
123
+ }
124
+ });
125
+ program
126
+ .command('snapshot')
127
+ .description('Manage evaluation snapshots')
128
+ .addCommand(new Command('create')
129
+ .description('Create snapshot from a run (only fully passing runs)')
130
+ .requiredOption('--run <runId>', 'Run id from runs/<runId>')
131
+ .option('--name <name>', 'Snapshot name')
132
+ .option('--runs-dir <path>', 'Directory with run artifacts', 'mcplab/results/evaluation-runs')
133
+ .option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
134
+ .action((options) => {
135
+ try {
136
+ const resultsPath = resolve(options.runsDir, String(options.run), 'results.json');
137
+ const results = JSON.parse(readFileSync(resultsPath, 'utf8'));
138
+ const snapshot = buildSnapshotFromRun(results, options.name);
139
+ const path = saveSnapshot(snapshot, resolve(options.snapshotsDir));
140
+ console.log(kleur.green(`Snapshot created: ${snapshot.id}`));
141
+ console.log(kleur.gray(`Path: ${path}`));
142
+ }
143
+ catch (err) {
144
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
145
+ process.exit(1);
146
+ }
147
+ }))
148
+ .addCommand(new Command('eval-init')
149
+ .description('Create baseline snapshot from a run and link it to config snapshot_eval policy')
150
+ .requiredOption('--run <runId>', 'Run id from runs/<runId>')
151
+ .requiredOption('--config <path>', 'Path to eval.yaml')
152
+ .option('--name <name>', 'Snapshot name')
153
+ .option('--runs-dir <path>', 'Directory with run artifacts', 'mcplab/results/evaluation-runs')
154
+ .option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
155
+ .action((options) => {
156
+ try {
157
+ const resultsPath = resolve(options.runsDir, String(options.run), 'results.json');
158
+ const results = JSON.parse(readFileSync(resultsPath, 'utf8'));
159
+ const snapshot = buildSnapshotFromRun(results, options.name);
160
+ saveSnapshot(snapshot, resolve(options.snapshotsDir));
161
+ const configPath = resolve(String(options.config));
162
+ const { sourceConfig } = loadConfig(configPath);
163
+ const nextConfig = {
164
+ ...sourceConfig,
165
+ snapshot_eval: {
166
+ enabled: true,
167
+ mode: sourceConfig.snapshot_eval?.mode ?? 'warn',
168
+ baseline_snapshot_id: snapshot.id,
169
+ baseline_source_run_id: results.metadata.run_id,
170
+ last_updated_at: new Date().toISOString()
171
+ }
172
+ };
173
+ writeFileSync(configPath, `${stringifyYaml(nextConfig)}\n`, 'utf8');
174
+ console.log(kleur.green(`Snapshot eval baseline linked: ${snapshot.id}`));
175
+ console.log(kleur.gray(`Config updated: ${configPath}`));
176
+ }
177
+ catch (err) {
178
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
179
+ process.exit(1);
180
+ }
181
+ }))
182
+ .addCommand(new Command('eval-policy')
183
+ .description('Update snapshot_eval policy in a config')
184
+ .requiredOption('--config <path>', 'Path to eval.yaml')
185
+ .requiredOption('--enabled <true|false>', 'Whether snapshot eval is enabled')
186
+ .requiredOption('--mode <warn|fail_on_drift>', 'Snapshot eval mode')
187
+ .option('--baseline-snapshot <snapshotId>', 'Baseline snapshot id')
188
+ .option('--baseline-source-run <runId>', 'Source run id used to create baseline')
189
+ .action((options) => {
190
+ try {
191
+ const enabled = String(options.enabled).toLowerCase() === 'true';
192
+ const mode = String(options.mode);
193
+ if (mode !== 'warn' && mode !== 'fail_on_drift') {
194
+ throw new Error('mode must be warn or fail_on_drift');
195
+ }
196
+ const configPath = resolve(String(options.config));
197
+ const { sourceConfig } = loadConfig(configPath);
198
+ const nextConfig = {
199
+ ...sourceConfig,
200
+ snapshot_eval: {
201
+ enabled,
202
+ mode,
203
+ baseline_snapshot_id: options.baselineSnapshot ?? sourceConfig.snapshot_eval?.baseline_snapshot_id,
204
+ baseline_source_run_id: options.baselineSourceRun ?? sourceConfig.snapshot_eval?.baseline_source_run_id,
205
+ last_updated_at: new Date().toISOString()
206
+ }
207
+ };
208
+ writeFileSync(configPath, `${stringifyYaml(nextConfig)}\n`, 'utf8');
209
+ console.log(kleur.green(`Snapshot eval policy updated: ${configPath}`));
210
+ }
211
+ catch (err) {
212
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
213
+ process.exit(1);
214
+ }
215
+ }))
216
+ .addCommand(new Command('eval-set-scenario')
217
+ .description('Set or clear a scenario-level snapshot baseline override in a config')
218
+ .requiredOption('--config <path>', 'Path to eval.yaml')
219
+ .requiredOption('--scenario <id>', 'Scenario id')
220
+ .option('--snapshot <snapshotId>', 'Override baseline snapshot id (omit to clear override)')
221
+ .option('--source-run <runId>', 'Source run id used to create the scenario baseline')
222
+ .option('--enabled <true|false>', 'Scenario snapshot eval enabled override')
223
+ .action((options) => {
224
+ try {
225
+ const configPath = resolve(String(options.config));
226
+ const scenarioId = String(options.scenario).trim();
227
+ if (!scenarioId)
228
+ throw new Error('scenario is required');
229
+ const { sourceConfig } = loadConfig(configPath);
230
+ const scenarios = [...(sourceConfig.scenarios ?? [])];
231
+ const scenarioIndex = scenarios.findIndex((s) => typeof s === 'object' && s !== null && !('ref' in s) && s.id === scenarioId);
232
+ if (scenarioIndex < 0) {
233
+ throw new Error(`Scenario not found in config.scenarios (inline only): ${scenarioId}`);
234
+ }
235
+ const current = scenarios[scenarioIndex];
236
+ if (!current || typeof current !== 'object' || 'ref' in current) {
237
+ throw new Error(`Scenario not found in config.scenarios (inline only): ${scenarioId}`);
238
+ }
239
+ const nextScenarioSnapshotEval = {
240
+ ...(current.snapshot_eval ?? {}),
241
+ ...(options.snapshot !== undefined
242
+ ? { baseline_snapshot_id: String(options.snapshot || '') || undefined }
243
+ : {}),
244
+ ...(options.sourceRun !== undefined
245
+ ? { baseline_source_run_id: String(options.sourceRun || '') || undefined }
246
+ : {}),
247
+ ...(options.enabled !== undefined
248
+ ? { enabled: String(options.enabled).toLowerCase() === 'true' }
249
+ : {}),
250
+ last_updated_at: new Date().toISOString()
251
+ };
252
+ if (!nextScenarioSnapshotEval.baseline_snapshot_id) {
253
+ delete nextScenarioSnapshotEval.baseline_snapshot_id;
254
+ }
255
+ if (!nextScenarioSnapshotEval.baseline_source_run_id) {
256
+ delete nextScenarioSnapshotEval.baseline_source_run_id;
257
+ }
258
+ if (nextScenarioSnapshotEval.enabled === undefined &&
259
+ !nextScenarioSnapshotEval.baseline_snapshot_id &&
260
+ !nextScenarioSnapshotEval.baseline_source_run_id) {
261
+ scenarios[scenarioIndex] = {
262
+ ...current,
263
+ snapshot_eval: undefined
264
+ };
265
+ }
266
+ else {
267
+ scenarios[scenarioIndex] = {
268
+ ...current,
269
+ snapshot_eval: nextScenarioSnapshotEval
270
+ };
271
+ }
272
+ const nextConfig = {
273
+ ...sourceConfig,
274
+ scenarios
275
+ };
276
+ writeFileSync(configPath, `${stringifyYaml(nextConfig)}\n`, 'utf8');
277
+ console.log(kleur.green(`Scenario snapshot baseline ${options.snapshot ? 'set' : 'updated'}: ${scenarioId}`));
278
+ console.log(kleur.gray(`Config updated: ${configPath}`));
279
+ }
280
+ catch (err) {
281
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
282
+ process.exit(1);
283
+ }
284
+ }))
285
+ .addCommand(new Command('list')
286
+ .description('List snapshots')
287
+ .option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
288
+ .action((options) => {
289
+ try {
290
+ const snapshots = listSnapshots(resolve(options.snapshotsDir));
291
+ if (snapshots.length === 0) {
292
+ console.log('No snapshots found.');
293
+ return;
294
+ }
295
+ for (const snapshot of snapshots) {
296
+ console.log(`${snapshot.id} ${snapshot.name} (run=${snapshot.source_run_id}, created=${snapshot.created_at})`);
297
+ }
298
+ }
299
+ catch (err) {
300
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
301
+ process.exit(1);
302
+ }
303
+ }))
304
+ .addCommand(new Command('show')
305
+ .description('Show snapshot JSON')
306
+ .requiredOption('--id <snapshotId>', 'Snapshot id')
307
+ .option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
308
+ .action((options) => {
309
+ try {
310
+ const snapshot = loadSnapshot(String(options.id), resolve(options.snapshotsDir));
311
+ console.log(JSON.stringify(snapshot, null, 2));
312
+ }
313
+ catch (err) {
314
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
315
+ process.exit(1);
316
+ }
317
+ }))
318
+ .addCommand(new Command('compare')
319
+ .description('Compare run against snapshot')
320
+ .requiredOption('--id <snapshotId>', 'Snapshot id')
321
+ .requiredOption('--run <runId>', 'Run id from runs/<runId>')
322
+ .option('--format <format>', 'Output format: table|json', 'table')
323
+ .option('--runs-dir <path>', 'Directory with run artifacts', 'mcplab/results/evaluation-runs')
324
+ .option('--snapshots-dir <path>', 'Directory for snapshots', 'mcplab/snapshots')
325
+ .action((options) => {
326
+ try {
327
+ const snapshot = loadSnapshot(String(options.id), resolve(options.snapshotsDir));
328
+ const resultsPath = resolve(options.runsDir, String(options.run), 'results.json');
329
+ const results = JSON.parse(readFileSync(resultsPath, 'utf8'));
330
+ const comparison = compareRunToSnapshot(results, snapshot);
331
+ if (String(options.format) === 'json') {
332
+ console.log(JSON.stringify(comparison, null, 2));
333
+ }
334
+ else {
335
+ console.log(formatSnapshotComparisonTable(comparison));
336
+ }
337
+ }
338
+ catch (err) {
339
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
340
+ process.exit(1);
341
+ }
342
+ }));
343
+ program
344
+ .command('migrate-configs')
345
+ .description('Migrate eval YAML files to the canonical list-based format')
346
+ .option('--evals-dir <path>', 'Directory for YAML evals', 'mcplab/evals')
347
+ .option('--dry-run', 'Preview migration without writing files')
348
+ .action((options) => {
349
+ try {
350
+ const evalsDir = resolve(String(options.evalsDir));
351
+ const bundleRoot = resolve(evalsDir, '..');
352
+ const files = readdirSync(evalsDir).filter((name) => name.endsWith('.yaml') || name.endsWith('.yml'));
353
+ let migrated = 0;
354
+ let skipped = 0;
355
+ let failed = 0;
356
+ for (const file of files) {
357
+ const filePath = resolve(evalsDir, file);
358
+ try {
359
+ const { sourceConfig, warnings } = loadConfig(filePath, { bundleRoot });
360
+ const hadLegacyServersMapWarning = warnings.some((warning) => warning.includes('Legacy servers object map was migrated'));
361
+ const hadLegacyAgentsMapWarning = warnings.some((warning) => warning.includes('Legacy agents object map was migrated'));
362
+ const hadLegacyInlineIdsWarning = warnings.some((warning) => warning.includes('Legacy inline server.name migrated') ||
363
+ warning.includes('Legacy inline agent.name migrated'));
364
+ if (!hadLegacyServersMapWarning &&
365
+ !hadLegacyAgentsMapWarning &&
366
+ !hadLegacyInlineIdsWarning) {
367
+ skipped += 1;
368
+ continue;
369
+ }
370
+ if (options.dryRun) {
371
+ console.log(kleur.cyan(`[dry-run] ${file}: would normalize config format${warnings.length ? ` (${warnings.join(' | ')})` : ''}`));
372
+ migrated += 1;
373
+ continue;
374
+ }
375
+ const nextConfig = { ...sourceConfig };
376
+ writeFileSync(filePath, `${stringifyYaml(nextConfig)}\n`, 'utf8');
377
+ migrated += 1;
378
+ console.log(kleur.green(`Migrated: ${file}`));
379
+ }
380
+ catch (error) {
381
+ failed += 1;
382
+ console.error(kleur.red(`Failed: ${file} (${error?.message ?? String(error)})`));
383
+ }
384
+ }
385
+ console.log(kleur.cyan(`Migration summary${options.dryRun ? ' (dry-run)' : ''}: migrated=${migrated}, skipped=${skipped}, failed=${failed}`));
386
+ }
387
+ catch (err) {
388
+ console.error(kleur.red(`Error: ${err?.message ?? String(err)}`));
389
+ process.exit(1);
390
+ }
391
+ });
392
+ program
393
+ .command('report')
394
+ .description('Regenerate report.html from a previous run')
395
+ .requiredOption('--input <runDir>', 'Run directory containing results.json')
396
+ .action((options) => {
397
+ try {
398
+ const runDir = resolve(options.input);
399
+ const resultsPath = join(runDir, 'results.json');
400
+ const reportPath = join(runDir, 'report.html');
401
+ const results = JSON.parse(readFileSync(resultsPath, 'utf8'));
402
+ const html = renderReport(results);
403
+ writeFileSync(reportPath, html, 'utf8');
404
+ console.log(kleur.green(`Report regenerated: ${reportPath}`));
405
+ }
406
+ catch (err) {
407
+ const message = err?.message ?? String(err);
408
+ console.error(kleur.red(`Error: ${message}`));
409
+ process.exit(1);
410
+ }
411
+ });
412
+ program
413
+ .command('app')
414
+ .description('Serve MCPLab app frontend and local API bridge')
415
+ .option('--evals-dir <path>', 'Directory for YAML evals', 'mcplab/evals')
416
+ .option('--runs-dir <path>', 'Directory for run artifacts', 'mcplab/results/evaluation-runs')
417
+ .option('--snapshots-dir <path>', 'Directory for snapshot artifacts', 'mcplab/snapshots')
418
+ .option('--tool-analysis-results-dir <path>', 'Directory for saved tool analysis reports', 'mcplab/results/tool-analysis')
419
+ .option('--libraries-dir <path>', 'Bundle root for reusable servers/agents/scenarios', 'mcplab')
420
+ .option('--port <number>', 'Port to bind', '8787')
421
+ .option('--host <host>', 'Host to bind', '127.0.0.1')
422
+ .option('--open', 'Open browser after startup')
423
+ .option('--dev', 'Proxy frontend requests to Vite dev server (API remains local)')
424
+ .action(async (options) => {
425
+ try {
426
+ const port = Number(options.port);
427
+ if (Number.isNaN(port) || port <= 0) {
428
+ throw new Error('Port must be a positive number');
429
+ }
430
+ await startAppServer({
431
+ host: options.host,
432
+ port,
433
+ evalsDir: resolve(options.evalsDir),
434
+ runsDir: resolve(options.runsDir),
435
+ snapshotsDir: resolve(options.snapshotsDir),
436
+ toolAnalysisResultsDir: resolve(options.toolAnalysisResultsDir),
437
+ librariesDir: resolve(options.librariesDir),
438
+ dev: Boolean(options.dev),
439
+ open: Boolean(options.open)
440
+ });
441
+ }
442
+ catch (err) {
443
+ const message = err?.message ?? String(err);
444
+ console.error(kleur.red(`Error: ${message}`));
445
+ process.exit(1);
446
+ }
447
+ });
448
+ program
449
+ .command('watch')
450
+ .description('Watch config file and auto-rerun evaluations on changes')
451
+ .requiredOption('-c, --config <path>', 'Path to eval.yaml')
452
+ .option('-s, --scenario <id>', 'Run a single scenario')
453
+ .option('-n, --runs <count>', 'Variance runs', '1')
454
+ .option('--runs-dir <path>', 'Directory for run artifacts', 'mcplab/results/evaluation-runs')
455
+ .option('--debounce <ms>', 'Debounce delay in milliseconds', '500')
456
+ .action(async (options) => {
457
+ const configPath = resolve(options.config);
458
+ const runsPerScenario = Number(options.runs);
459
+ const debounceMs = Number(options.debounce);
460
+ if (Number.isNaN(runsPerScenario) || runsPerScenario <= 0) {
461
+ console.error(kleur.red('Error: Runs must be a positive number'));
462
+ process.exit(1);
463
+ }
464
+ console.log(kleur.cyan(`👀 Watching: ${configPath}`));
465
+ console.log(kleur.gray(`Press Ctrl+C to stop\n`));
466
+ let running = false;
467
+ let debounceTimer = null;
468
+ const runEvaluation = async () => {
469
+ if (running) {
470
+ console.log(kleur.yellow('⏭️ Evaluation already running, skipping...'));
471
+ return;
472
+ }
473
+ running = true;
474
+ const timestamp = new Date().toLocaleTimeString();
475
+ console.log(kleur.cyan(`\n⚡ [${timestamp}] Running evaluation...`));
476
+ try {
477
+ const { config, hash } = loadConfig(configPath);
478
+ const expanded = expandConfigForAgents(config, config.run_defaults?.selected_agents);
479
+ const selected = selectScenarios(expanded, options.scenario);
480
+ const { runDir, results } = await runAll(selected, {
481
+ runsPerScenario,
482
+ scenarioId: options.scenario,
483
+ configHash: hash,
484
+ gitCommit: getGitCommit(),
485
+ cliVersion: pkgVersion,
486
+ runsDir: String(options.runsDir)
487
+ });
488
+ const reportPath = join(runDir, 'report.html');
489
+ writeFileSync(reportPath, renderReport(results), 'utf8');
490
+ console.log(kleur.green(`✅ Run completed: ${runDir}`));
491
+ }
492
+ catch (err) {
493
+ const message = err?.message ?? String(err);
494
+ const hint = message.includes('fetch failed')
495
+ ? ' Hint: verify the MCP server is running, the SSE URL is correct, and any bearer token env var is set.'
496
+ : '';
497
+ console.error(kleur.red(`❌ Error: ${message}${hint}`));
498
+ }
499
+ finally {
500
+ running = false;
501
+ }
502
+ };
503
+ // Initial run
504
+ await runEvaluation();
505
+ // Watch for changes
506
+ const watcher = chokidar.watch(configPath, {
507
+ persistent: true,
508
+ ignoreInitial: true,
509
+ awaitWriteFinish: {
510
+ stabilityThreshold: 100,
511
+ pollInterval: 50
512
+ }
513
+ });
514
+ watcher.on('change', () => {
515
+ if (debounceTimer) {
516
+ clearTimeout(debounceTimer);
517
+ }
518
+ debounceTimer = setTimeout(() => {
519
+ runEvaluation();
520
+ }, debounceMs);
521
+ });
522
+ watcher.on('error', (error) => {
523
+ console.error(kleur.red(`Watcher error: ${error}`));
524
+ });
525
+ // Handle graceful shutdown
526
+ process.on('SIGINT', () => {
527
+ console.log(kleur.cyan('\n\n👋 Stopping watcher...'));
528
+ watcher.close();
529
+ process.exit(0);
530
+ });
531
+ });
532
+ program.parse();
533
+ function getGitCommit() {
534
+ try {
535
+ const output = execSync('git rev-parse HEAD', { stdio: ['ignore', 'pipe', 'ignore'] })
536
+ .toString()
537
+ .trim();
538
+ return output || undefined;
539
+ }
540
+ catch {
541
+ return undefined;
542
+ }
543
+ }
544
+ //# sourceMappingURL=cli.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EACL,UAAU,EACV,eAAe,EACf,MAAM,EACN,qBAAqB,EAKtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EACL,8BAA8B,EAC9B,oBAAoB,EACpB,oBAAoB,EACpB,6BAA6B,EAC7B,aAAa,EACb,YAAY,EACZ,YAAY,EACb,MAAM,eAAe,CAAC;AAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9F,EAAE,OAAiB,CAAC;AAEtB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAC9B,OAAO;KACJ,IAAI,CAAC,QAAQ,CAAC;KACd,WAAW,CAAC,uDAAuD,CAAC;KACpE,OAAO,CAAC,UAAU,CAAC,CAAC;AAEvB,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,0BAA0B,CAAC;KACvC,cAAc,CAAC,qBAAqB,EAAE,mBAAmB,CAAC;KAC1D,MAAM,CAAC,qBAAqB,EAAE,uBAAuB,CAAC;KACtD,MAAM,CAAC,oBAAoB,EAAE,eAAe,EAAE,GAAG,CAAC;KAClD,MAAM,CACL,mBAAmB,EACnB,6EAA6E,CAC9E;KACA,MAAM,CAAC,iBAAiB,EAAE,qDAAqD,CAAC;KAChF,MAAM,CAAC,iCAAiC,EAAE,2CAA2C,CAAC;KACtF,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,EAAE,gCAAgC,CAAC;KAC5F,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,kBAAkB,CAAC;KAC/E,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACrE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM;YACpC,CAAC,CAAC,OAAO,CAAC,MAAM;iBACX,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBAC5B,MAAM,CAAC,OAAO,CAAC;YACpB,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,iBAAiB,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;QAClD,MAAM,eAAe,GAAG,eAAe,IAAI,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC;QAChF,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAChE,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,KAAK,iBAAiB,IAAI,eAAe,EAAE,MAAM,EAAE,CAAC;YAC/E,MAAM,UAAU,GAAG,eAAe,EAAE,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;YAChF,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CACR,cAAc,iBAAiB,gBAAgB,UAAU,sBAAsB,QAAQ,CAAC,SAAS,CAAC,MAAM,cAAc,CACvH,CACF,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE;YACjD,eAAe;YACf,UAAU,EAAE,OAAO,CAAC,QAAQ;YAC5B,UAAU,EAAE,IAAI;YAChB,SAAS,EAAE,YAAY,EAAE;YACzB,UAAU,EAAE,UAAU;YACtB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SACjC,CAAC,CAAC;QACH,IAAI,iBAAiB,GAAG,KAAK,CAAC;QAC9B,MAAM,eAAe,GACnB,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAE1E,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,oBAAoB,EAAE,CAAC;gBAClC,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,MAAM,CAAC,iEAAiE,CAAC,CAChF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,GAAG,YAAY,CAC3B,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,EACnC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC3D,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAChC,QAAQ,CAAC,SAAS;qBACf,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,KAAK,KAAK,CAAC;qBAC/D,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAClC,CAAC;gBACF,MAAM,OAAO,GAAG,8BAA8B,CAAC;oBAC7C,OAAO;oBACP,WAAW,EAAE,CAAC,UAAU,CAAC;oBACzB,MAAM;oBACN,kBAAkB;iBACnB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CACT,GAAG,OAAO,CAAC,IAAI,eAAe,OAAO,CAAC,oBAAoB,cAAc,OAAO,CAAC,aAAa,aAAa,OAAO,CAAC,MAAM,EAAE,CAC3H,CAAC;gBACF,IAAI,OAAO,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1C,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,MAAM,CAAC,uBAAuB,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAC7E,CAAC;gBACJ,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,UAAU,CAAC,CAAC,CAAC;gBACvD,iBAAiB;oBACf,MAAM,CAAC,IAAI,KAAK,eAAe,IAAI,OAAO,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACjD,aAAa,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5E,aAAa,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,MAAM,EAAE,CAAC,CAAC,CAAC;QAErD,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,YAAY,CAC3B,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAC/B,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAC9B,CAAC;YACF,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,UAAU,CAAC,CAAC,CAAC;QACzD,CAAC;QAED,uCAAuC;QACvC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC,CAAC;YACvD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,2CAA2C,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC,CACtF,CAAC;QACJ,CAAC;QACD,IAAI,iBAAiB,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC3C,CAAC,CAAC,uGAAuG;YACzG,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,6BAA6B,CAAC;KAC1C,UAAU,CACT,IAAI,OAAO,CAAC,QAAQ,CAAC;KAClB,WAAW,CAAC,sDAAsD,CAAC;KACnE,cAAc,CAAC,eAAe,EAAE,0BAA0B,CAAC;KAC3D,MAAM,CAAC,eAAe,EAAE,eAAe,CAAC;KACxC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,EAAE,gCAAgC,CAAC;KAC7F,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,kBAAkB,CAAC;KAC/E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,qBAAqB,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL;KACA,UAAU,CACT,IAAI,OAAO,CAAC,WAAW,CAAC;KACrB,WAAW,CAAC,gFAAgF,CAAC;KAC7F,cAAc,CAAC,eAAe,EAAE,0BAA0B,CAAC;KAC3D,cAAc,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;KACtD,MAAM,CAAC,eAAe,EAAE,eAAe,CAAC;KACxC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,EAAE,gCAAgC,CAAC;KAC7F,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,kBAAkB,CAAC;KAC/E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAgB,CAAC;QAC7E,MAAM,QAAQ,GAAG,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7D,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAEtD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,EAAE,YAAY,EAAE,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,UAAU,GAAqB;YACnC,GAAG,YAAY;YACf,aAAa,EAAE;gBACb,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,YAAY,CAAC,aAAa,EAAE,IAAI,IAAI,MAAM;gBAChD,oBAAoB,EAAE,QAAQ,CAAC,EAAE;gBACjC,sBAAsB,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAC/C,eAAe,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aAC1C;SACF,CAAC;QACF,aAAa,CAAC,UAAU,EAAE,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAEpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,kCAAkC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL;KACA,UAAU,CACT,IAAI,OAAO,CAAC,aAAa,CAAC;KACvB,WAAW,CAAC,yCAAyC,CAAC;KACtD,cAAc,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;KACtD,cAAc,CAAC,wBAAwB,EAAE,kCAAkC,CAAC;KAC5E,cAAc,CAAC,6BAA6B,EAAE,oBAAoB,CAAC;KACnE,MAAM,CAAC,kCAAkC,EAAE,sBAAsB,CAAC;KAClE,MAAM,CAAC,+BAA+B,EAAE,uCAAuC,CAAC;KAChF,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;QACjE,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAA6B,CAAC;QAC9D,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,EAAE,YAAY,EAAE,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,UAAU,GAAqB;YACnC,GAAG,YAAY;YACf,aAAa,EAAE;gBACb,OAAO;gBACP,IAAI;gBACJ,oBAAoB,EAClB,OAAO,CAAC,gBAAgB,IAAI,YAAY,CAAC,aAAa,EAAE,oBAAoB;gBAC9E,sBAAsB,EACpB,OAAO,CAAC,iBAAiB,IAAI,YAAY,CAAC,aAAa,EAAE,sBAAsB;gBACjF,eAAe,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aAC1C;SACF,CAAC;QACF,aAAa,CAAC,UAAU,EAAE,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,iCAAiC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL;KACA,UAAU,CACT,IAAI,OAAO,CAAC,mBAAmB,CAAC;KAC7B,WAAW,CAAC,sEAAsE,CAAC;KACnF,cAAc,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;KACtD,cAAc,CAAC,iBAAiB,EAAE,aAAa,CAAC;KAChD,MAAM,CAAC,yBAAyB,EAAE,wDAAwD,CAAC;KAC3F,MAAM,CAAC,sBAAsB,EAAE,oDAAoD,CAAC;KACpF,MAAM,CAAC,wBAAwB,EAAE,yCAAyC,CAAC;KAC3E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACzD,MAAM,EAAE,YAAY,EAAE,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,UAAU,CACnF,CAAC;QACF,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,yDAAyD,UAAU,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,OAAO,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CAAC,yDAAyD,UAAU,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,wBAAwB,GAAG;YAC/B,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;YAChC,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS;gBAChC,CAAC,CAAC,EAAE,oBAAoB,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,SAAS,EAAE;gBACvE,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;gBACjC,CAAC,CAAC,EAAE,sBAAsB,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,IAAI,SAAS,EAAE;gBAC1E,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS;gBAC/B,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE;gBAC/D,CAAC,CAAC,EAAE,CAAC;YACP,eAAe,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAC1C,CAAC;QACF,IAAI,CAAC,wBAAwB,CAAC,oBAAoB,EAAE,CAAC;YACnD,OAAQ,wBAAgC,CAAC,oBAAoB,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,EAAE,CAAC;YACrD,OAAQ,wBAAgC,CAAC,sBAAsB,CAAC;QAClE,CAAC;QACD,IACE,wBAAwB,CAAC,OAAO,KAAK,SAAS;YAC9C,CAAC,wBAAwB,CAAC,oBAAoB;YAC9C,CAAC,wBAAwB,CAAC,sBAAsB,EAChD,CAAC;YACD,SAAS,CAAC,aAAa,CAAC,GAAG;gBACzB,GAAG,OAAO;gBACV,aAAa,EAAE,SAAS;aACzB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,aAAa,CAAC,GAAG;gBACzB,GAAG,OAAO;gBACV,aAAa,EAAE,wBAAwB;aACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAqB;YACnC,GAAG,YAAY;YACf,SAAS;SACV,CAAC;QACF,aAAa,CAAC,UAAU,EAAE,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CACT,8BAA8B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,EAAE,CACpF,CACF,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL;KACA,UAAU,CACT,IAAI,OAAO,CAAC,MAAM,CAAC;KAChB,WAAW,CAAC,gBAAgB,CAAC;KAC7B,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,kBAAkB,CAAC;KAC/E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAC/D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CACT,GAAG,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,IAAI,UAAU,QAAQ,CAAC,aAAa,aAAa,QAAQ,CAAC,UAAU,GAAG,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL;KACA,UAAU,CACT,IAAI,OAAO,CAAC,MAAM,CAAC;KAChB,WAAW,CAAC,oBAAoB,CAAC;KACjC,cAAc,CAAC,mBAAmB,EAAE,aAAa,CAAC;KAClD,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,kBAAkB,CAAC;KAC/E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACjF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL;KACA,UAAU,CACT,IAAI,OAAO,CAAC,SAAS,CAAC;KACnB,WAAW,CAAC,8BAA8B,CAAC;KAC3C,cAAc,CAAC,mBAAmB,EAAE,aAAa,CAAC;KAClD,cAAc,CAAC,eAAe,EAAE,0BAA0B,CAAC;KAC3D,MAAM,CAAC,mBAAmB,EAAE,2BAA2B,EAAE,OAAO,CAAC;KACjE,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,EAAE,gCAAgC,CAAC;KAC7F,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,kBAAkB,CAAC;KAC/E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC3D,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,UAAU,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CACL,CAAC;AAEJ,OAAO;KACJ,OAAO,CAAC,iBAAiB,CAAC;KAC1B,WAAW,CAAC,4DAA4D,CAAC;KACzE,MAAM,CAAC,oBAAoB,EAAE,0BAA0B,EAAE,cAAc,CAAC;KACxE,MAAM,CAAC,WAAW,EAAE,yCAAyC,CAAC;KAC9D,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACtG,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;gBACxE,MAAM,0BAA0B,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC3D,OAAO,CAAC,QAAQ,CAAC,wCAAwC,CAAC,CAC3D,CAAC;gBACF,MAAM,yBAAyB,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC1D,OAAO,CAAC,QAAQ,CAAC,uCAAuC,CAAC,CAC1D,CAAC;gBACF,MAAM,yBAAyB,GAAG,QAAQ,CAAC,IAAI,CAC7C,CAAC,OAAO,EAAE,EAAE,CACV,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC;oBACtD,OAAO,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CACxD,CAAC;gBAEF,IACE,CAAC,0BAA0B;oBAC3B,CAAC,yBAAyB;oBAC1B,CAAC,yBAAyB,EAC1B,CAAC;oBACD,OAAO,IAAI,CAAC,CAAC;oBACb,SAAS;gBACX,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CACR,aAAa,IAAI,kCAAkC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACzG,CACF,CAAC;oBACF,QAAQ,IAAI,CAAC,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,MAAM,UAAU,GAAqB,EAAE,GAAG,YAAY,EAAE,CAAC;gBACzD,aAAa,CAAC,QAAQ,EAAE,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAClE,QAAQ,IAAI,CAAC,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,IAAI,KAAK,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CACR,oBAAoB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,cAAc,QAAQ,aAAa,OAAO,YAAY,MAAM,EAAE,CACrH,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,4CAA4C,CAAC;KACzD,cAAc,CAAC,kBAAkB,EAAE,uCAAuC,CAAC;KAC3E,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACnC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,oBAAoB,EAAE,0BAA0B,EAAE,cAAc,CAAC;KACxE,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,EAAE,gCAAgC,CAAC;KAC5F,MAAM,CAAC,wBAAwB,EAAE,kCAAkC,EAAE,kBAAkB,CAAC;KACxF,MAAM,CACL,oCAAoC,EACpC,2CAA2C,EAC3C,8BAA8B,CAC/B;KACA,MAAM,CAAC,wBAAwB,EAAE,mDAAmD,EAAE,QAAQ,CAAC;KAC/F,MAAM,CAAC,iBAAiB,EAAE,cAAc,EAAE,MAAM,CAAC;KACjD,MAAM,CAAC,eAAe,EAAE,cAAc,EAAE,WAAW,CAAC;KACpD,MAAM,CAAC,QAAQ,EAAE,4BAA4B,CAAC;KAC9C,MAAM,CAAC,OAAO,EAAE,gEAAgE,CAAC;KACjF,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,CAAC;YACnB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI;YACJ,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;YACnC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;YACjC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;YAC3C,sBAAsB,EAAE,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC;YAC/D,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;YAC3C,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC;YACzB,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;SAC5B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,yDAAyD,CAAC;KACtE,cAAc,CAAC,qBAAqB,EAAE,mBAAmB,CAAC;KAC1D,MAAM,CAAC,qBAAqB,EAAE,uBAAuB,CAAC;KACtD,MAAM,CAAC,oBAAoB,EAAE,eAAe,EAAE,GAAG,CAAC;KAClD,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,EAAE,gCAAgC,CAAC;KAC5F,MAAM,CAAC,iBAAiB,EAAE,gCAAgC,EAAE,KAAK,CAAC;KAClE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE5C,IAAI,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;QAC1D,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAElD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,aAAa,GAA0B,IAAI,CAAC;IAEhD,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;QAC/B,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,6CAA6C,CAAC,CAAC,CAAC;YACzE,OAAO;QACT,CAAC;QAED,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,SAAS,yBAAyB,CAAC,CAAC,CAAC;QAEpE,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;YACrF,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7D,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE;gBACjD,eAAe;gBACf,UAAU,EAAE,OAAO,CAAC,QAAQ;gBAC5B,UAAU,EAAE,IAAI;gBAChB,SAAS,EAAE,YAAY,EAAE;gBACzB,UAAU,EAAE,UAAU;gBACtB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;aACjC,CAAC,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YAC/C,aAAa,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,oBAAoB,MAAM,EAAE,CAAC,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAC3C,CAAC,CAAC,uGAAuG;gBACzG,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;QACzD,CAAC;gBAAS,CAAC;YACT,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;IACH,CAAC,CAAC;IAEF,cAAc;IACd,MAAM,aAAa,EAAE,CAAC;IAEtB,oBAAoB;IACpB,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE;QACzC,UAAU,EAAE,IAAI;QAChB,aAAa,EAAE,IAAI;QACnB,gBAAgB,EAAE;YAChB,kBAAkB,EAAE,GAAG;YACvB,YAAY,EAAE,EAAE;SACjB;KACF,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,IAAI,aAAa,EAAE,CAAC;YAClB,YAAY,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QACD,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,aAAa,EAAE,CAAC;QAClB,CAAC,EAAE,UAAU,CAAC,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QAC5B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;QACtD,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC;AAEhB,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;aACnF,QAAQ,EAAE;aACV,IAAI,EAAE,CAAC;QACV,OAAO,MAAM,IAAI,SAAS,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
@@ -0,0 +1,80 @@
1
+ import type { ResultsJson } from '@inspectr/mcplab-core';
2
+ export interface SnapshotSourceSummary {
3
+ total_scenarios: number;
4
+ total_runs: number;
5
+ pass_rate: number;
6
+ }
7
+ export interface SnapshotItem {
8
+ scenario_id: string;
9
+ baseline_agents: string[];
10
+ required_tools: string[];
11
+ forbidden_tools: string[];
12
+ allowed_sequences: string[][];
13
+ baseline_tools: string[];
14
+ extracted_values: Record<string, string | number | boolean | null>;
15
+ final_answer_features: {
16
+ token_set: string[];
17
+ };
18
+ }
19
+ export interface SnapshotRecord {
20
+ schema_version: 2;
21
+ id: string;
22
+ name: string;
23
+ created_at: string;
24
+ source_run_id: string;
25
+ config_hash: string;
26
+ source_summary: SnapshotSourceSummary;
27
+ items: SnapshotItem[];
28
+ }
29
+ export interface ScenarioComparison {
30
+ scenario_id: string;
31
+ baseline_agents: string[];
32
+ observed_agents: string[];
33
+ score: number;
34
+ status: 'Match' | 'Warn' | 'Drift';
35
+ components: {
36
+ tools: number;
37
+ extracts: number;
38
+ semantics: number;
39
+ };
40
+ reasons: string[];
41
+ }
42
+ export interface SnapshotComparison {
43
+ snapshot_id: string;
44
+ run_id: string;
45
+ overall_score: number;
46
+ scenario_results: ScenarioComparison[];
47
+ }
48
+ export interface SnapshotEvalPolicy {
49
+ enabled: boolean;
50
+ mode: 'warn' | 'fail_on_drift';
51
+ baseline_snapshot_id?: string;
52
+ baseline_source_run_id?: string;
53
+ last_updated_at?: string;
54
+ }
55
+ export interface AppliedSnapshotEval {
56
+ applied: boolean;
57
+ mode: 'warn' | 'fail_on_drift';
58
+ baseline_snapshot_id: string;
59
+ baseline_source_run_id?: string;
60
+ overall_score: number;
61
+ status: 'Match' | 'Warn' | 'Drift';
62
+ impacted_scenarios: string[];
63
+ }
64
+ export declare function ensureRunFullyPassing(results: ResultsJson): void;
65
+ export declare function buildSnapshotFromRun(results: ResultsJson, name?: string): SnapshotRecord;
66
+ export declare function compareRunToSnapshot(
67
+ results: ResultsJson,
68
+ snapshot: SnapshotRecord
69
+ ): SnapshotComparison;
70
+ export declare function applySnapshotPolicyToRunResult(params: {
71
+ results: ResultsJson;
72
+ comparisons: SnapshotComparison[];
73
+ policy: SnapshotEvalPolicy;
74
+ enabledScenarioIds?: Set<string>;
75
+ }): AppliedSnapshotEval;
76
+ export declare function saveSnapshot(record: SnapshotRecord, snapshotsDir: string): string;
77
+ export declare function loadSnapshot(id: string, snapshotsDir: string): SnapshotRecord;
78
+ export declare function listSnapshots(snapshotsDir: string): SnapshotRecord[];
79
+ export declare function formatSnapshotComparisonTable(comparison: SnapshotComparison): string;
80
+ //# sourceMappingURL=snapshot.d.ts.map