@inspectr/mcplab 1.15.0 → 1.17.0

package/dist/app-server/runs-routes.js

@@ -2,33 +2,108 @@ import { randomUUID } from 'node:crypto';
 import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { isAbsolute, join, relative, resolve } from 'node:path';
-import { McpClientManager, loadConfig, hashConfig, runAll, renderSummaryMarkdown } from '@inspectr/mcplab-core';
+import { McpClientManager, loadConfig, hashConfig, runAll, renderSummaryMarkdown, applyRuntimeServerOverrides } from '@inspectr/mcplab-core';
 import { renderReport } from '@inspectr/mcplab-reporting';
 import { OAuthAuthorizationRequiredError } from './oauth-session-manager.js';
+import { selectScenarioIds } from './runs-store.js';
+import { readLibraries as readLibrariesFromStore } from './libraries-store.js';
+export function mergeLibraryEntriesIntoConfig(config, libraryAgents, libraryServers) {
+    return {
+        ...config,
+        agents: { ...libraryAgents, ...config.agents },
+        servers: { ...libraryServers, ...config.servers }
+    };
+}
+export function applyLibraryEntries(loaded, libraryAgents, libraryServers) {
+    loaded.config = mergeLibraryEntriesIntoConfig(loaded.config, libraryAgents, libraryServers);
+    loaded.hash = hashConfig(loaded.config);
+}
+function filterScenarioOverridesToSelectedScenarios(selectedConfig, scenarioServerOverrides) {
+    if (!scenarioServerOverrides)
+        return undefined;
+    const selectedIds = new Set(selectedConfig.scenarios.map((scenario) => scenario.id));
+    const filtered = Object.fromEntries(Object.entries(scenarioServerOverrides).filter(([scenarioId]) => selectedIds.has(scenarioId)));
+    return Object.keys(filtered).length > 0 ? filtered : undefined;
+}
+// Backward-compatible exports used by existing tests/imports.
 export function mergeLibraryAgentsIntoConfig(config, libraryAgents) {
-    return { ...config, agents: { ...libraryAgents, ...config.agents } };
+    return mergeLibraryEntriesIntoConfig(config, libraryAgents, {});
 }
 export function applyLibraryAgents(loaded, libraryAgents) {
-    loaded.config = mergeLibraryAgentsIntoConfig(loaded.config, libraryAgents);
-    loaded.hash = hashConfig(loaded.config);
+    applyLibraryEntries(loaded, libraryAgents, {});
 }
 export async function handleRunsRoutes(params) {
     const { req, res, pathname, method, settings, jobs, runQueueState, oauthSessionManager, deps } = params;
-    const { parseBody, asJson, addJobEvent, sendSseEvent, ensureInsideRoot, listRuns, getRunResults, getScenarioRunTraceRecords, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, loadSnapshot, compareRunToSnapshot, applySnapshotPolicyToRunResult, readLibraries, pickDefaultAssistantAgentName, pkgVersion } = deps;
+    const { parseBody, asJson, addJobEvent, sendSseEvent, ensureInsideRoot, listRuns, getRunResults, getScenarioRunTraceRecords, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, readLibraries, pickDefaultAssistantAgentName, pkgVersion } = deps;
     if (pathname === '/api/runs' && method === 'GET') {
         const requestUrl = new URL(req.url ?? '/api/runs', 'http://localhost');
         const since = requestUrl.searchParams.get('since') ?? undefined;
         const until = requestUrl.searchParams.get('until') ?? undefined;
+        const scenario = requestUrl.searchParams.get('scenario') ?? undefined;
         const lastDaysRaw = requestUrl.searchParams.get('last_days');
         const lastDaysParsed = lastDaysRaw === null ? NaN : Number(lastDaysRaw);
         const lastDays = Number.isFinite(lastDaysParsed) && lastDaysParsed > 0
             ? Math.floor(lastDaysParsed)
             : undefined;
-        asJson(res, 200, listRuns(settings.runsDir, {
+        const limitRaw = Number(requestUrl.searchParams.get('limit'));
+        const offsetRaw = Number(requestUrl.searchParams.get('offset'));
+        const limit = Number.isFinite(limitRaw) ? Math.max(1, Math.min(100, Math.floor(limitRaw))) : 25;
+        const offset = Number.isFinite(offsetRaw) ? Math.max(0, Math.floor(offsetRaw)) : 0;
+        const all = listRuns(settings.runsDir, {
             since,
             until,
-            lastDays
-        }));
+            lastDays,
+            scenario
+        });
+        const data = all.slice(offset, offset + limit);
+        const totalCount = all.length;
+        const hasMore = offset + data.length < totalCount;
+        const nextOffset = hasMore ? offset + data.length : null;
+        const prevOffset = offset > 0 ? Math.max(0, offset - limit) : null;
+        asJson(res, 200, {
+            object: 'list',
+            url: `${pathname}${requestUrl.search}`,
+            data,
+            has_more: hasMore,
+            total_count: totalCount,
+            next_offset: nextOffset,
+            prev_offset: prevOffset
+        });
+        return true;
+    }
+    if (pathname === '/api/runs/latest-pass-rates' && method === 'POST') {
+        const body = (await parseBody(req));
+        const requestedConfigs = Array.isArray(body.configs) ? body.configs : [];
+        const normalizedConfigs = requestedConfigs
+            .map((entry) => ({
+            id: String(entry?.id ?? '').trim(),
+            sourcePath: String(entry?.sourcePath ?? '').trim(),
+            relativePath: String(entry?.relativePath ?? '').trim(),
+            configHash: String(entry?.configHash ?? '').trim()
+        }))
+            .filter((entry) => entry.id);
+        const lastDaysRaw = Number(body.lastDays);
+        const lastDays = Number.isFinite(lastDaysRaw) && lastDaysRaw > 0 ? Math.floor(lastDaysRaw) : undefined;
+        const summaries = listRuns(settings.runsDir, { lastDays });
+        const pending = new Set(normalizedConfigs.map((entry) => entry.id));
+        const byConfigId = {};
+        for (const summary of summaries) {
+            if (pending.size === 0)
+                break;
+            const summaryPath = String(summary.configPath ?? '').trim();
+            const summaryHash = String(summary.configHash ?? '').trim();
+            for (const cfg of normalizedConfigs) {
+                if (!pending.has(cfg.id))
+                    continue;
+                if ((cfg.sourcePath && cfg.sourcePath === summaryPath) ||
+                    (cfg.relativePath && cfg.relativePath === summaryPath) ||
+                    (cfg.configHash && cfg.configHash === summaryHash)) {
+                    byConfigId[cfg.id] = summary.passRate;
+                    pending.delete(cfg.id);
+                }
+            }
+        }
+        asJson(res, 200, { byConfigId });
         return true;
     }
     if (pathname.startsWith('/api/runs/') && pathname.endsWith('/trace') && method === 'GET') {
@@ -109,7 +184,9 @@ export async function handleRunsRoutes(params) {
                 runsPerScenario: j.runParams.runsPerScenario,
                 scenarioIds: j.runParams.scenarioIds ?? null,
                 agents: j.runParams.requestedAgents ?? null,
-                runNote: j.runParams.runNote ?? null
+                runNote: j.runParams.runNote ?? null,
+                serverOverrideAll: j.runParams.serverOverrideAll ?? null,
+                scenarioServerOverrides: j.runParams.scenarioServerOverrides ?? null
             }
         }));
         asJson(res, 200, {
@@ -122,7 +199,9 @@ export async function handleRunsRoutes(params) {
                         runsPerScenario: activeJob.runParams.runsPerScenario,
                         scenarioIds: activeJob.runParams.scenarioIds ?? null,
                         agents: activeJob.runParams.requestedAgents ?? null,
-                        runNote: activeJob.runParams.runNote ?? null
+                        runNote: activeJob.runParams.runNote ?? null,
+                        serverOverrideAll: activeJob.runParams.serverOverrideAll ?? null,
+                        scenarioServerOverrides: activeJob.runParams.scenarioServerOverrides ?? null
                     }
                 }
                 : null,
@@ -182,9 +261,45 @@ export async function handleRunsRoutes(params) {
         const requestedAgents = Array.isArray(body.agents)
             ? body.agents.map((agent) => String(agent).trim()).filter(Boolean)
             : undefined;
-        const applySnapshotEval = body.applySnapshotEval !== false;
         const runNoteRaw = typeof body.runNote === 'string' ? body.runNote.trim() : '';
         const runNote = runNoteRaw ? runNoteRaw.slice(0, 500) : undefined;
+        const serverOverrideAll = Array.isArray(body.serverOverrideAll)
+            ? body.serverOverrideAll.map((id) => String(id).trim()).filter(Boolean)
+            : undefined;
+        if (Array.isArray(body.serverOverrideAll) &&
+            (!serverOverrideAll || serverOverrideAll.length === 0)) {
+            asJson(res, 400, { error: 'serverOverrideAll must include at least one server id' });
+            return true;
+        }
+        if (body.scenarioServerOverrides !== undefined &&
+            (typeof body.scenarioServerOverrides !== 'object' ||
+                body.scenarioServerOverrides === null ||
+                Array.isArray(body.scenarioServerOverrides))) {
+            asJson(res, 400, {
+                error: 'scenarioServerOverrides must be an object of scenarioId -> string[]'
+            });
+            return true;
+        }
+        let scenarioServerOverrides;
+        if (body.scenarioServerOverrides && typeof body.scenarioServerOverrides === 'object') {
+            const normalizedEntries = [];
+            for (const [rawScenarioId, rawServerIds] of Object.entries(body.scenarioServerOverrides)) {
+                const scenarioOverrideId = String(rawScenarioId).trim();
+                if (!scenarioOverrideId)
+                    continue;
+                if (!Array.isArray(rawServerIds)) {
+                    asJson(res, 400, {
+                        error: `scenarioServerOverrides.${scenarioOverrideId} must be an array of server ids`
+                    });
+                    return true;
+                }
+                normalizedEntries.push([
+                    scenarioOverrideId,
+                    rawServerIds.map((id) => String(id).trim()).filter(Boolean)
+                ]);
+            }
+            scenarioServerOverrides = Object.fromEntries(normalizedEntries);
+        }
         if (!configPathRaw) {
             asJson(res, 400, { error: 'configPath is required' });
             return true;
@@ -200,17 +315,27 @@ export async function handleRunsRoutes(params) {
             asJson(res, 404, { error: `Config not found: ${configPath}` });
             return true;
         }
-        // Eagerly cache OAuth server names to avoid re-parsing config in advanceQueue
-        let oauthServerNames;
         try {
-            const loaded = loadConfig(configPath);
-            oauthServerNames = Object.entries(loaded.config.servers ?? {})
-                .filter(([, v]) => v.auth?.type === 'oauth_authorization_code')
-                .map(([name]) => name);
+            const loaded = loadConfig(configPath, { bundleRoot: settings.librariesDir });
+            const libraries = readLibraries(settings.librariesDir);
+            applyLibraryEntries(loaded, libraries.agents, libraries.servers);
+            const selected = scenarioIds?.length
+                ? deps.selectScenarioIds(loaded.config, scenarioIds)
+                : scenarioId
+                    ? deps.selectScenarioIds(loaded.config, [scenarioId])
+                    : loaded.config;
+            const filteredScenarioOverrides = filterScenarioOverridesToSelectedScenarios(selected, scenarioServerOverrides);
+            applyRuntimeServerOverrides(selected, {
+                serverOverrideAll,
+                scenarioServerOverrides: filteredScenarioOverrides
+            });
         }
-        catch {
-            // Will be resolved lazily in advanceQueue if needed
+        catch (error) {
+            asJson(res, 400, { error: error instanceof Error ? error.message : String(error) });
+            return true;
         }
+        // Resolve lazily in advanceQueue so runtime overrides are always reflected.
+        const oauthServerNames = undefined;
         const jobId = `run-${Date.now()}-${randomUUID().slice(0, 8)}`;
         const runParamsObj = {
             configPath,
@@ -218,9 +343,10 @@ export async function handleRunsRoutes(params) {
             scenarioId,
             scenarioIds,
             requestedAgents,
-            applySnapshotEval,
             runNote,
-            oauthServerNames
+            oauthServerNames,
+            serverOverrideAll,
+            scenarioServerOverrides
         };
         const job = {
             id: jobId,
@@ -244,6 +370,8 @@ export async function handleRunsRoutes(params) {
                     scenarioIds: scenarioIds ?? null,
                     agents: requestedAgents ?? null,
                     runNote: runNote ?? null,
+                    serverOverrideAll: serverOverrideAll ?? null,
+                    scenarioServerOverrides: scenarioServerOverrides ?? null,
                     position: runQueueState.queue.length
                 }
             });
@@ -568,19 +696,39 @@ function toCoreExtractRules(extractRules) {
     }
     return rules;
 }
-function resolveOAuthServersForJob(job) {
-    if (job.runParams.oauthServerNames !== undefined) {
+function resolveOAuthServersForJob(job, librariesDir) {
+    if (job.runParams.oauthServerNames !== undefined)
         return job.runParams.oauthServerNames;
-    }
     try {
-        const loaded = loadConfig(job.runParams.configPath);
-        const names = Object.entries(loaded.config.servers ?? {})
-            .filter(([, v]) => v.auth?.type === 'oauth_authorization_code')
-            .map(([name]) => name);
+        const loaded = loadConfig(job.runParams.configPath, { bundleRoot: librariesDir });
+        const libraries = readLibrariesFromStore(librariesDir);
+        applyLibraryEntries(loaded, libraries.agents, libraries.servers);
+        const selected = job.runParams.scenarioIds?.length
+            ? selectScenarioIds(loaded.config, job.runParams.scenarioIds)
+            : job.runParams.scenarioId
+                ? selectScenarioIds(loaded.config, [job.runParams.scenarioId])
+                : loaded.config;
+        const filteredScenarioOverrides = filterScenarioOverridesToSelectedScenarios(selected, job.runParams.scenarioServerOverrides);
+        const withOverrides = applyRuntimeServerOverrides(selected, {
+            serverOverrideAll: job.runParams.serverOverrideAll,
+            scenarioServerOverrides: filteredScenarioOverrides
+        });
+        const effectiveServers = new Set(withOverrides.scenarios.flatMap((scenario) => scenario.servers));
+        const names = Array.from(effectiveServers).filter((name) => {
+            const config = withOverrides.servers?.[name];
+            return config?.auth?.type === 'oauth_authorization_code';
+        });
         job.runParams.oauthServerNames = names;
         return names;
     }
-    catch {
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (message.includes('Unknown server refs') ||
+            message.includes('Unknown scenarios in scenarioServerOverrides') ||
+            message.includes('serverOverrideAll must include at least one server id')) {
+            throw error;
+        }
+        console.warn(`[mcplab] Failed to resolve OAuth servers for queued job '${job.id}': ${message}`);
         return [];
     }
 }
@@ -599,7 +747,25 @@ async function advanceQueue(jobs, runQueueState, settings, oauthSessionManager,
                 continue;
             }
             // Pre-check OAuth before starting
-            const oauthServers = resolveOAuthServersForJob(nextJob);
+            let oauthServers = [];
+            try {
+                oauthServers = resolveOAuthServersForJob(nextJob, settings.librariesDir);
+            }
+            catch (error) {
+                runQueueState.queue.shift();
+                nextJob.status = 'error';
+                deps.addJobEvent(nextJob, {
+                    type: 'error',
+                    ts: new Date().toISOString(),
+                    payload: {
+                        message: error instanceof Error ? error.message : String(error)
+                    }
+                });
+                for (const client of nextJob.clients)
+                    client.end();
+                nextJob.clients.clear();
+                continue;
+            }
             if (oauthServers.length > 0) {
                 const authStatus = oauthSessionManager.checkServersAuthStatus(oauthServers);
                 const needsAuth = authStatus.filter((s) => s.status === 'auth_required');
@@ -634,7 +800,9 @@ async function advanceQueue(jobs, runQueueState, settings, oauthSessionManager,
                     scenarioId: nextJob.runParams.scenarioId ?? null,
                     scenarioIds: nextJob.runParams.scenarioIds ?? null,
                     agents: nextJob.runParams.requestedAgents ?? null,
-                    runNote: nextJob.runParams.runNote ?? null
+                    runNote: nextJob.runParams.runNote ?? null,
+                    serverOverrideAll: nextJob.runParams.serverOverrideAll ?? null,
+                    scenarioServerOverrides: nextJob.runParams.scenarioServerOverrides ?? null
                 }
             });
             void executeRunJob(nextJob, settings, jobs, runQueueState, oauthSessionManager, deps);
@@ -646,8 +814,8 @@ async function advanceQueue(jobs, runQueueState, settings, oauthSessionManager,
     }
 }
 async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionManager, deps) {
-    const { addJobEvent, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, loadSnapshot, compareRunToSnapshot, applySnapshotPolicyToRunResult, readLibraries, pkgVersion } = deps;
-    const { configPath, runsPerScenario, scenarioId, scenarioIds, requestedAgents, applySnapshotEval, runNote } = job.runParams;
+    const { addJobEvent, getScenarioRunTraceRecords, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, readLibraries, pkgVersion } = deps;
+    const { configPath, runsPerScenario, scenarioId, scenarioIds, requestedAgents, runNote, serverOverrideAll, scenarioServerOverrides } = job.runParams;
     try {
         addJobEvent(job, {
             type: 'log',
@@ -655,8 +823,8 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
             payload: { message: `Loading MCP Evaluation config: ${configPath}` }
         });
         const loaded = loadConfig(configPath, { bundleRoot: settings.librariesDir });
-        const { agents: libraryAgents } = readLibraries(settings.librariesDir);
-        applyLibraryAgents(loaded, libraryAgents);
+        const { agents: libraryAgents, servers: libraryServers } = readLibraries(settings.librariesDir);
+        applyLibraryEntries(loaded, libraryAgents, libraryServers);
         addJobEvent(job, {
             type: 'log',
             ts: new Date().toISOString(),
@@ -690,7 +858,30 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 message: `Selected ${selectedBaseScenarios.scenarios.length} base scenario(s)`
             }
         });
-        const resolvedAgents = resolveRunSelectedAgents(selectedBaseScenarios, requestedAgents);
+        const filteredScenarioOverrides = filterScenarioOverridesToSelectedScenarios(selectedBaseScenarios, scenarioServerOverrides);
+        const runtimeOverriddenConfig = applyRuntimeServerOverrides(selectedBaseScenarios, {
+            serverOverrideAll,
+            scenarioServerOverrides: filteredScenarioOverrides
+        });
+        const effectiveConfigHash = hashConfig(runtimeOverriddenConfig);
+        addJobEvent(job, {
+            type: 'log',
+            ts: new Date().toISOString(),
+            payload: {
+                message: `Applied runtime server overrides: global=${serverOverrideAll?.length ?? 0} scenario-specific=${Object.keys(filteredScenarioOverrides ?? {}).length}`
+            }
+        });
+        const effectiveScenarioServers = runtimeOverriddenConfig.scenarios
+            .map((scenario) => `${scenario.id}=[${scenario.servers.join(', ')}]`)
+            .join('; ');
+        addJobEvent(job, {
+            type: 'log',
+            ts: new Date().toISOString(),
+            payload: {
+                message: `Effective MCP servers per scenario: ${effectiveScenarioServers || '(none)'}`
+            }
+        });
+        const resolvedAgents = resolveRunSelectedAgents(runtimeOverriddenConfig, requestedAgents);
         const resolvedAgentList = Array.isArray(resolvedAgents) ? resolvedAgents : [];
         addJobEvent(job, {
             type: 'log',
@@ -701,7 +892,7 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                     : `Using resolved default agents: ${resolvedAgentList.join(', ')}`
             }
         });
-        const expandedConfig = expandConfigForAgents(selectedBaseScenarios, resolvedAgents);
+        const expandedConfig = expandConfigForAgents(runtimeOverriddenConfig, resolvedAgents);
         addJobEvent(job, {
             type: 'log',
             ts: new Date().toISOString(),
@@ -709,9 +900,8 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 message: `Expanded to ${expandedConfig.scenarios.length} executable scenario run(s) across selected agents`
             }
         });
-        const oauthServers = Object.entries(expandedConfig.servers)
-            .filter(([, serverConfig]) => serverConfig.auth?.type === 'oauth_authorization_code')
-            .map(([serverName]) => serverName);
+        const usedServerNames = new Set(expandedConfig.scenarios.flatMap((scenario) => scenario.servers));
+        const oauthServers = Array.from(usedServerNames).filter((serverName) => expandedConfig.servers[serverName]?.auth?.type === 'oauth_authorization_code');
         const mcpServerAuthHeaders = oauthServers.length > 0
             ? await oauthSessionManager.getAuthHeadersForServers(oauthServers)
             : undefined;
@@ -745,7 +935,7 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 runsPerScenario,
                 scenarioId,
                 runNote,
-                configHash: loaded.hash,
+                configHash: effectiveConfigHash,
                 cliVersion: pkgVersion,
                 runsDir: settings.runsDir,
                 mcpServerAuthHeaders,
@@ -767,6 +957,23 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
             if (loaded.config.name && loaded.config.name.trim().length > 0) {
                 results.metadata.config_name = loaded.config.name.trim();
             }
+            results.metadata.rerun_agents = [...resolvedAgentList];
+            results.metadata.rerun_scenario_ids = selectedBaseScenarios.scenarios.map((scenario) => scenario.id);
+            if (serverOverrideAll && serverOverrideAll.length > 0) {
+                results.metadata.rerun_server_override_all = [...serverOverrideAll];
+            }
+            else {
+                delete results.metadata.rerun_server_override_all;
+            }
+            if (filteredScenarioOverrides && Object.keys(filteredScenarioOverrides).length > 0) {
+                results.metadata.rerun_scenario_server_overrides = Object.fromEntries(Object.entries(filteredScenarioOverrides).map(([scenarioKey, serverIds]) => [
+                    scenarioKey,
+                    [...serverIds]
+                ]));
+            }
+            else {
+                delete results.metadata.rerun_scenario_server_overrides;
+            }
             addJobEvent(job, {
                 type: 'log',
                 ts: new Date().toISOString(),
@@ -774,102 +981,13 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                     message: `Evaluation execution finished (run id: ${results.metadata.run_id})`
                 }
             });
-            if (applySnapshotEval && expandedConfig.snapshot_eval?.enabled) {
-                addJobEvent(job, {
-                    type: 'log',
-                    ts: new Date().toISOString(),
-                    payload: { message: 'Applying snapshot evaluation policy ...' }
-                });
-                const policy = expandedConfig.snapshot_eval;
-                const enabledScenarioIds = new Set(selectedBaseScenarios.scenarios
-                    .filter((scenario) => scenario.snapshot_eval?.enabled !== false)
-                    .map((scenario) => scenario.id));
-                const scenarioBaselineMap = new Map();
-                for (const scenario of selectedBaseScenarios.scenarios) {
-                    if (scenario.snapshot_eval?.enabled === false)
-                        continue;
-                    const baselineId = scenario.snapshot_eval?.baseline_snapshot_id ?? policy.baseline_snapshot_id;
-                    if (baselineId)
-                        scenarioBaselineMap.set(scenario.id, baselineId);
-                }
-                const scenariosWithoutBaseline = selectedBaseScenarios.scenarios
-                    .filter((scenario) => scenario.snapshot_eval?.enabled !== false)
-                    .filter((scenario) => !(scenario.snapshot_eval?.baseline_snapshot_id ?? policy.baseline_snapshot_id))
-                    .map((scenario) => scenario.id);
-                if (scenariosWithoutBaseline.length > 0) {
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: `Snapshot eval enabled but no baseline configured for scenarios: ${scenariosWithoutBaseline.join(', ')}`
-                        }
-                    });
-                }
-                const comparisons = [];
-                const scenarioIdsByBaseline = new Map();
-                for (const [scenarioIdItem, baselineId] of scenarioBaselineMap) {
-                    const list = scenarioIdsByBaseline.get(baselineId) ?? [];
-                    list.push(scenarioIdItem);
-                    scenarioIdsByBaseline.set(baselineId, list);
-                }
-                for (const [baselineId, scenarioIdsForBaseline] of scenarioIdsByBaseline) {
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: `Comparing ${scenarioIdsForBaseline.length} scenario(s) to snapshot baseline '${baselineId}'`
-                        }
-                    });
-                    const snapshot = loadSnapshot(baselineId, settings.snapshotsDir);
-                    const fullComparison = compareRunToSnapshot(results, snapshot);
-                    comparisons.push({
-                        ...fullComparison,
-                        scenario_results: fullComparison.scenario_results.filter((row) => scenarioIdsForBaseline.includes(row.scenario_id))
-                    });
-                }
-                if (comparisons.length > 0) {
-                    applySnapshotPolicyToRunResult({ results, comparisons, policy, enabledScenarioIds });
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: `Snapshot evaluation applied (${comparisons.length} baseline comparison group(s))`
-                        }
-                    });
-                }
-                else {
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: 'Snapshot evaluation enabled, but no baseline comparisons were applied'
-                        }
-                    });
-                }
-            }
-            else if (applySnapshotEval) {
-                addJobEvent(job, {
-                    type: 'log',
-                    ts: new Date().toISOString(),
-                    payload: {
-                        message: 'Snapshot evaluation requested, but config snapshot evaluation is disabled'
-                    }
-                });
-            }
-            else {
-                addJobEvent(job, {
-                    type: 'log',
-                    ts: new Date().toISOString(),
-                    payload: {
-                        message: 'Snapshot evaluation skipped for this run (disabled in run request)'
-                    }
-                });
-            }
             addJobEvent(job, {
                 type: 'log',
                 ts: new Date().toISOString(),
                 payload: { message: `Writing results to ${runDir}` }
             });
+            const traceRecords = getScenarioRunTraceRecords(results.metadata.run_id, settings.runsDir);
+            results.metadata.tool_tokens_total = estimateRunToolTokensTotal(traceRecords);
             writeFileSync(join(runDir, 'results.json'), `${JSON.stringify(results, null, 2)}\n`, 'utf8');
             writeFileSync(join(runDir, 'report.html'), renderReport(results), 'utf8');
             writeFileSync(join(runDir, 'summary.md'), renderSummaryMarkdown(results), 'utf8');
@@ -886,8 +1004,7 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 payload: {
                     runId: results.metadata.run_id,
                     runDir,
-                    summary: results.summary,
-                    snapshotEval: results.metadata.snapshot_eval ?? null
+                    summary: results.summary
                 }
             });
             job.status = 'completed';
@@ -923,6 +1040,77 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
         pruneOldJobs(jobs, runQueueState);
     }
 }
+function splitInteger(total, parts) {
+    if (!Number.isFinite(total) || !parts || parts <= 0)
+        return Array(parts).fill(0);
+    const safeTotal = Math.max(0, Math.round(total ?? 0));
+    const base = Math.floor(safeTotal / parts);
+    let remainder = safeTotal % parts;
+    return Array.from({ length: parts }, () => {
+        const value = base + (remainder > 0 ? 1 : 0);
+        if (remainder > 0)
+            remainder -= 1;
+        return value;
+    });
+}
+function estimateRunToolTokensTotal(records) {
+    let total = 0;
+    let hasAny = false;
+    for (const record of records) {
+        const toolUsesById = new Map();
+        for (const message of record.messages ?? []) {
+            const toolUses = message.content.filter((block) => block.type === 'tool_use');
+            if (toolUses.length > 0) {
+                for (const toolUse of toolUses)
+                    toolUsesById.set(toolUse.id, toolUse.name);
+                const allEstimated = toolUses.every((toolUse) => Boolean(toolUse.estimated_tokens));
+                if (allEstimated) {
+                    for (const toolUse of toolUses)
+                        total += toolUse.estimated_tokens?.total ?? 0;
+                    hasAny = true;
+                }
+                else if (toolUses.length === 1 && typeof message.usage?.total_tokens === 'number') {
+                    total += message.usage.total_tokens;
+                    hasAny = true;
+                }
+                else {
+                    const shares = splitInteger(message.usage?.total_tokens, toolUses.length);
+                    total += shares.reduce((sum, value) => sum + value, 0);
+                    if (typeof message.usage?.total_tokens === 'number')
+                        hasAny = true;
+                }
+            }
+            const toolResults = message.content.filter((block) => block.type === 'tool_result');
+            if (toolResults.length === 0)
+                continue;
+            const allEstimated = toolResults.every((result) => Boolean(result.estimated_tokens));
+            if (allEstimated) {
+                for (const result of toolResults)
+                    total += result.estimated_tokens?.total ?? 0;
+                hasAny = true;
+                continue;
+            }
+            if (toolResults.length === 1) {
+                const [result] = toolResults;
+                if (result &&
+                    toolUsesById.has(result.tool_use_id) &&
+                    typeof message.usage?.total_tokens === 'number') {
+                    total += message.usage.total_tokens;
+                    hasAny = true;
+                    continue;
+                }
+            }
+            const knownResults = toolResults.filter((result) => toolUsesById.has(result.tool_use_id));
+            if (knownResults.length === 0)
+                continue;
+            const shares = splitInteger(message.usage?.total_tokens, knownResults.length);
+            total += shares.reduce((sum, value) => sum + value, 0);
+            if (typeof message.usage?.total_tokens === 'number')
+                hasAny = true;
+        }
+    }
+    return hasAny ? total : null;
+}
 function pruneOldJobs(jobs, runQueueState) {
     const maxAgeMs = 30 * 60_000;
     const now = Date.now();
@@ -945,9 +1133,9 @@ function formatRunProgressMessage(event) {
         case 'run_started':
             return `Run initialized (id: ${event.runId}, ${event.totalScenarioRuns} scenario run(s))`;
         case 'mcp_connect_started':
-            return `Connecting to ${event.serverCount} MCP server(s) ...`;
+            return `Connecting to ${event.serverCount} MCP server(s): ${event.serverNames.join(', ')} ...`;
         case 'mcp_connect_finished':
-            return `Connected to ${event.serverCount} MCP server(s)`;
+            return `Connected to ${event.serverCount} MCP server(s): ${event.serverNames.join(', ')}`;
         case 'scenario_run_started':
             return `Scenario ${event.scenarioRunIndex}/${event.totalScenarioRuns} started: ${event.scenarioId} [agent=${event.agentName}, run=${event.runIndex + 1}/${event.runsPerScenario}]`;
         case 'scenario_run_finished':