@inspectr/mcplab 1.14.3 → 1.16.0

package/dist/app-server/runs-routes.js

@@ -2,21 +2,53 @@ import { randomUUID } from 'node:crypto';
 import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { isAbsolute, join, relative, resolve } from 'node:path';
-import { McpClientManager, loadConfig, hashConfig, runAll, renderSummaryMarkdown } from '@inspectr/mcplab-core';
+import { McpClientManager, loadConfig, hashConfig, runAll, renderSummaryMarkdown, applyRuntimeServerOverrides } from '@inspectr/mcplab-core';
 import { renderReport } from '@inspectr/mcplab-reporting';
 import { OAuthAuthorizationRequiredError } from './oauth-session-manager.js';
+import { selectScenarioIds } from './runs-store.js';
+import { readLibraries as readLibrariesFromStore } from './libraries-store.js';
+export function mergeLibraryEntriesIntoConfig(config, libraryAgents, libraryServers) {
+    return {
+        ...config,
+        agents: { ...libraryAgents, ...config.agents },
+        servers: { ...libraryServers, ...config.servers }
+    };
+}
+export function applyLibraryEntries(loaded, libraryAgents, libraryServers) {
+    loaded.config = mergeLibraryEntriesIntoConfig(loaded.config, libraryAgents, libraryServers);
+    loaded.hash = hashConfig(loaded.config);
+}
+function filterScenarioOverridesToSelectedScenarios(selectedConfig, scenarioServerOverrides) {
+    if (!scenarioServerOverrides)
+        return undefined;
+    const selectedIds = new Set(selectedConfig.scenarios.map((scenario) => scenario.id));
+    const filtered = Object.fromEntries(Object.entries(scenarioServerOverrides).filter(([scenarioId]) => selectedIds.has(scenarioId)));
+    return Object.keys(filtered).length > 0 ? filtered : undefined;
+}
+// Backward-compatible exports used by existing tests/imports.
 export function mergeLibraryAgentsIntoConfig(config, libraryAgents) {
-    return { ...config, agents: { ...libraryAgents, ...config.agents } };
+    return mergeLibraryEntriesIntoConfig(config, libraryAgents, {});
 }
 export function applyLibraryAgents(loaded, libraryAgents) {
-    loaded.config = mergeLibraryAgentsIntoConfig(loaded.config, libraryAgents);
-    loaded.hash = hashConfig(loaded.config);
+    applyLibraryEntries(loaded, libraryAgents, {});
 }
 export async function handleRunsRoutes(params) {
     const { req, res, pathname, method, settings, jobs, runQueueState, oauthSessionManager, deps } = params;
-    const { parseBody, asJson, addJobEvent, sendSseEvent, ensureInsideRoot, listRuns, getRunResults, getScenarioRunTraceRecords, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, loadSnapshot, compareRunToSnapshot, applySnapshotPolicyToRunResult, readLibraries, pickDefaultAssistantAgentName, pkgVersion } = deps;
+    const { parseBody, asJson, addJobEvent, sendSseEvent, ensureInsideRoot, listRuns, getRunResults, getScenarioRunTraceRecords, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, readLibraries, pickDefaultAssistantAgentName, pkgVersion } = deps;
     if (pathname === '/api/runs' && method === 'GET') {
-        asJson(res, 200, listRuns(settings.runsDir));
+        const requestUrl = new URL(req.url ?? '/api/runs', 'http://localhost');
+        const since = requestUrl.searchParams.get('since') ?? undefined;
+        const until = requestUrl.searchParams.get('until') ?? undefined;
+        const lastDaysRaw = requestUrl.searchParams.get('last_days');
+        const lastDaysParsed = lastDaysRaw === null ? NaN : Number(lastDaysRaw);
+        const lastDays = Number.isFinite(lastDaysParsed) && lastDaysParsed > 0
+            ? Math.floor(lastDaysParsed)
+            : undefined;
+        asJson(res, 200, listRuns(settings.runsDir, {
+            since,
+            until,
+            lastDays
+        }));
         return true;
     }
     if (pathname.startsWith('/api/runs/') && pathname.endsWith('/trace') && method === 'GET') {
@@ -97,7 +129,9 @@ export async function handleRunsRoutes(params) {
                 runsPerScenario: j.runParams.runsPerScenario,
                 scenarioIds: j.runParams.scenarioIds ?? null,
                 agents: j.runParams.requestedAgents ?? null,
-                runNote: j.runParams.runNote ?? null
+                runNote: j.runParams.runNote ?? null,
+                serverOverrideAll: j.runParams.serverOverrideAll ?? null,
+                scenarioServerOverrides: j.runParams.scenarioServerOverrides ?? null
             }
         }));
         asJson(res, 200, {
@@ -110,7 +144,9 @@ export async function handleRunsRoutes(params) {
                         runsPerScenario: activeJob.runParams.runsPerScenario,
                         scenarioIds: activeJob.runParams.scenarioIds ?? null,
                         agents: activeJob.runParams.requestedAgents ?? null,
-                        runNote: activeJob.runParams.runNote ?? null
+                        runNote: activeJob.runParams.runNote ?? null,
+                        serverOverrideAll: activeJob.runParams.serverOverrideAll ?? null,
+                        scenarioServerOverrides: activeJob.runParams.scenarioServerOverrides ?? null
                     }
                 }
                 : null,
@@ -170,9 +206,45 @@ export async function handleRunsRoutes(params) {
         const requestedAgents = Array.isArray(body.agents)
             ? body.agents.map((agent) => String(agent).trim()).filter(Boolean)
             : undefined;
-        const applySnapshotEval = body.applySnapshotEval !== false;
         const runNoteRaw = typeof body.runNote === 'string' ? body.runNote.trim() : '';
         const runNote = runNoteRaw ? runNoteRaw.slice(0, 500) : undefined;
+        const serverOverrideAll = Array.isArray(body.serverOverrideAll)
+            ? body.serverOverrideAll.map((id) => String(id).trim()).filter(Boolean)
+            : undefined;
+        if (Array.isArray(body.serverOverrideAll) &&
+            (!serverOverrideAll || serverOverrideAll.length === 0)) {
+            asJson(res, 400, { error: 'serverOverrideAll must include at least one server id' });
+            return true;
+        }
+        if (body.scenarioServerOverrides !== undefined &&
+            (typeof body.scenarioServerOverrides !== 'object' ||
+                body.scenarioServerOverrides === null ||
+                Array.isArray(body.scenarioServerOverrides))) {
+            asJson(res, 400, {
+                error: 'scenarioServerOverrides must be an object of scenarioId -> string[]'
+            });
+            return true;
+        }
+        let scenarioServerOverrides;
+        if (body.scenarioServerOverrides && typeof body.scenarioServerOverrides === 'object') {
+            const normalizedEntries = [];
+            for (const [rawScenarioId, rawServerIds] of Object.entries(body.scenarioServerOverrides)) {
+                const scenarioOverrideId = String(rawScenarioId).trim();
+                if (!scenarioOverrideId)
+                    continue;
+                if (!Array.isArray(rawServerIds)) {
+                    asJson(res, 400, {
+                        error: `scenarioServerOverrides.${scenarioOverrideId} must be an array of server ids`
+                    });
+                    return true;
+                }
+                normalizedEntries.push([
+                    scenarioOverrideId,
+                    rawServerIds.map((id) => String(id).trim()).filter(Boolean)
+                ]);
+            }
+            scenarioServerOverrides = Object.fromEntries(normalizedEntries);
+        }
         if (!configPathRaw) {
             asJson(res, 400, { error: 'configPath is required' });
             return true;
@@ -188,17 +260,27 @@ export async function handleRunsRoutes(params) {
             asJson(res, 404, { error: `Config not found: ${configPath}` });
             return true;
         }
-        // Eagerly cache OAuth server names to avoid re-parsing config in advanceQueue
-        let oauthServerNames;
         try {
-            const loaded = loadConfig(configPath);
-            oauthServerNames = Object.entries(loaded.config.servers ?? {})
-                .filter(([, v]) => v.auth?.type === 'oauth_authorization_code')
-                .map(([name]) => name);
+            const loaded = loadConfig(configPath, { bundleRoot: settings.librariesDir });
+            const libraries = readLibraries(settings.librariesDir);
+            applyLibraryEntries(loaded, libraries.agents, libraries.servers);
+            const selected = scenarioIds?.length
+                ? deps.selectScenarioIds(loaded.config, scenarioIds)
+                : scenarioId
+                    ? deps.selectScenarioIds(loaded.config, [scenarioId])
+                    : loaded.config;
+            const filteredScenarioOverrides = filterScenarioOverridesToSelectedScenarios(selected, scenarioServerOverrides);
+            applyRuntimeServerOverrides(selected, {
+                serverOverrideAll,
+                scenarioServerOverrides: filteredScenarioOverrides
+            });
         }
-        catch {
-            // Will be resolved lazily in advanceQueue if needed
+        catch (error) {
+            asJson(res, 400, { error: error instanceof Error ? error.message : String(error) });
+            return true;
         }
+        // Resolve lazily in advanceQueue so runtime overrides are always reflected.
+        const oauthServerNames = undefined;
         const jobId = `run-${Date.now()}-${randomUUID().slice(0, 8)}`;
         const runParamsObj = {
             configPath,
@@ -206,9 +288,10 @@ export async function handleRunsRoutes(params) {
             scenarioId,
             scenarioIds,
             requestedAgents,
-            applySnapshotEval,
             runNote,
-            oauthServerNames
+            oauthServerNames,
+            serverOverrideAll,
+            scenarioServerOverrides
         };
         const job = {
             id: jobId,
@@ -232,6 +315,8 @@ export async function handleRunsRoutes(params) {
                     scenarioIds: scenarioIds ?? null,
                     agents: requestedAgents ?? null,
                     runNote: runNote ?? null,
+                    serverOverrideAll: serverOverrideAll ?? null,
+                    scenarioServerOverrides: scenarioServerOverrides ?? null,
                     position: runQueueState.queue.length
                 }
             });
@@ -556,19 +641,39 @@ function toCoreExtractRules(extractRules) {
     }
     return rules;
 }
-function resolveOAuthServersForJob(job) {
-    if (job.runParams.oauthServerNames !== undefined) {
+function resolveOAuthServersForJob(job, librariesDir) {
+    if (job.runParams.oauthServerNames !== undefined)
         return job.runParams.oauthServerNames;
-    }
     try {
-        const loaded = loadConfig(job.runParams.configPath);
-        const names = Object.entries(loaded.config.servers ?? {})
-            .filter(([, v]) => v.auth?.type === 'oauth_authorization_code')
-            .map(([name]) => name);
+        const loaded = loadConfig(job.runParams.configPath, { bundleRoot: librariesDir });
+        const libraries = readLibrariesFromStore(librariesDir);
+        applyLibraryEntries(loaded, libraries.agents, libraries.servers);
+        const selected = job.runParams.scenarioIds?.length
+            ? selectScenarioIds(loaded.config, job.runParams.scenarioIds)
+            : job.runParams.scenarioId
+                ? selectScenarioIds(loaded.config, [job.runParams.scenarioId])
+                : loaded.config;
+        const filteredScenarioOverrides = filterScenarioOverridesToSelectedScenarios(selected, job.runParams.scenarioServerOverrides);
+        const withOverrides = applyRuntimeServerOverrides(selected, {
+            serverOverrideAll: job.runParams.serverOverrideAll,
+            scenarioServerOverrides: filteredScenarioOverrides
+        });
+        const effectiveServers = new Set(withOverrides.scenarios.flatMap((scenario) => scenario.servers));
+        const names = Array.from(effectiveServers).filter((name) => {
+            const config = withOverrides.servers?.[name];
+            return config?.auth?.type === 'oauth_authorization_code';
+        });
         job.runParams.oauthServerNames = names;
         return names;
     }
-    catch {
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (message.includes('Unknown server refs') ||
+            message.includes('Unknown scenarios in scenarioServerOverrides') ||
+            message.includes('serverOverrideAll must include at least one server id')) {
+            throw error;
+        }
+        console.warn(`[mcplab] Failed to resolve OAuth servers for queued job '${job.id}': ${message}`);
         return [];
     }
 }
@@ -587,7 +692,25 @@ async function advanceQueue(jobs, runQueueState, settings, oauthSessionManager,
                 continue;
             }
             // Pre-check OAuth before starting
-            const oauthServers = resolveOAuthServersForJob(nextJob);
+            let oauthServers = [];
+            try {
+                oauthServers = resolveOAuthServersForJob(nextJob, settings.librariesDir);
+            }
+            catch (error) {
+                runQueueState.queue.shift();
+                nextJob.status = 'error';
+                deps.addJobEvent(nextJob, {
+                    type: 'error',
+                    ts: new Date().toISOString(),
+                    payload: {
+                        message: error instanceof Error ? error.message : String(error)
+                    }
+                });
+                for (const client of nextJob.clients)
+                    client.end();
+                nextJob.clients.clear();
+                continue;
+            }
             if (oauthServers.length > 0) {
                 const authStatus = oauthSessionManager.checkServersAuthStatus(oauthServers);
                 const needsAuth = authStatus.filter((s) => s.status === 'auth_required');
@@ -622,7 +745,9 @@ async function advanceQueue(jobs, runQueueState, settings, oauthSessionManager,
                     scenarioId: nextJob.runParams.scenarioId ?? null,
                     scenarioIds: nextJob.runParams.scenarioIds ?? null,
                     agents: nextJob.runParams.requestedAgents ?? null,
-                    runNote: nextJob.runParams.runNote ?? null
+                    runNote: nextJob.runParams.runNote ?? null,
+                    serverOverrideAll: nextJob.runParams.serverOverrideAll ?? null,
+                    scenarioServerOverrides: nextJob.runParams.scenarioServerOverrides ?? null
                 }
             });
             void executeRunJob(nextJob, settings, jobs, runQueueState, oauthSessionManager, deps);
@@ -634,8 +759,8 @@ async function advanceQueue(jobs, runQueueState, settings, oauthSessionManager,
     }
 }
 async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionManager, deps) {
-    const { addJobEvent, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, loadSnapshot, compareRunToSnapshot, applySnapshotPolicyToRunResult, readLibraries, pkgVersion } = deps;
-    const { configPath, runsPerScenario, scenarioId, scenarioIds, requestedAgents, applySnapshotEval, runNote } = job.runParams;
+    const { addJobEvent, selectScenarioIds, expandConfigForAgents, resolveRunSelectedAgents, readLibraries, pkgVersion } = deps;
+    const { configPath, runsPerScenario, scenarioId, scenarioIds, requestedAgents, runNote, serverOverrideAll, scenarioServerOverrides } = job.runParams;
     try {
         addJobEvent(job, {
             type: 'log',
@@ -643,8 +768,8 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
             payload: { message: `Loading MCP Evaluation config: ${configPath}` }
         });
         const loaded = loadConfig(configPath, { bundleRoot: settings.librariesDir });
-        const { agents: libraryAgents } = readLibraries(settings.librariesDir);
-        applyLibraryAgents(loaded, libraryAgents);
+        const { agents: libraryAgents, servers: libraryServers } = readLibraries(settings.librariesDir);
+        applyLibraryEntries(loaded, libraryAgents, libraryServers);
         addJobEvent(job, {
             type: 'log',
             ts: new Date().toISOString(),
@@ -678,7 +803,30 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 message: `Selected ${selectedBaseScenarios.scenarios.length} base scenario(s)`
             }
         });
-        const resolvedAgents = resolveRunSelectedAgents(selectedBaseScenarios, requestedAgents);
+        const filteredScenarioOverrides = filterScenarioOverridesToSelectedScenarios(selectedBaseScenarios, scenarioServerOverrides);
+        const runtimeOverriddenConfig = applyRuntimeServerOverrides(selectedBaseScenarios, {
+            serverOverrideAll,
+            scenarioServerOverrides: filteredScenarioOverrides
+        });
+        const effectiveConfigHash = hashConfig(runtimeOverriddenConfig);
+        addJobEvent(job, {
+            type: 'log',
+            ts: new Date().toISOString(),
+            payload: {
+                message: `Applied runtime server overrides: global=${serverOverrideAll?.length ?? 0} scenario-specific=${Object.keys(filteredScenarioOverrides ?? {}).length}`
+            }
+        });
+        const effectiveScenarioServers = runtimeOverriddenConfig.scenarios
+            .map((scenario) => `${scenario.id}=[${scenario.servers.join(', ')}]`)
+            .join('; ');
+        addJobEvent(job, {
+            type: 'log',
+            ts: new Date().toISOString(),
+            payload: {
+                message: `Effective MCP servers per scenario: ${effectiveScenarioServers || '(none)'}`
+            }
+        });
+        const resolvedAgents = resolveRunSelectedAgents(runtimeOverriddenConfig, requestedAgents);
         const resolvedAgentList = Array.isArray(resolvedAgents) ? resolvedAgents : [];
         addJobEvent(job, {
             type: 'log',
@@ -689,7 +837,7 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                     : `Using resolved default agents: ${resolvedAgentList.join(', ')}`
             }
         });
-        const expandedConfig = expandConfigForAgents(selectedBaseScenarios, resolvedAgents);
+        const expandedConfig = expandConfigForAgents(runtimeOverriddenConfig, resolvedAgents);
         addJobEvent(job, {
             type: 'log',
             ts: new Date().toISOString(),
@@ -697,9 +845,8 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 message: `Expanded to ${expandedConfig.scenarios.length} executable scenario run(s) across selected agents`
             }
         });
-        const oauthServers = Object.entries(expandedConfig.servers)
-            .filter(([, serverConfig]) => serverConfig.auth?.type === 'oauth_authorization_code')
-            .map(([serverName]) => serverName);
+        const usedServerNames = new Set(expandedConfig.scenarios.flatMap((scenario) => scenario.servers));
+        const oauthServers = Array.from(usedServerNames).filter((serverName) => expandedConfig.servers[serverName]?.auth?.type === 'oauth_authorization_code');
         const mcpServerAuthHeaders = oauthServers.length > 0
             ? await oauthSessionManager.getAuthHeadersForServers(oauthServers)
             : undefined;
@@ -733,7 +880,7 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 runsPerScenario,
                 scenarioId,
                 runNote,
-                configHash: loaded.hash,
+                configHash: effectiveConfigHash,
                 cliVersion: pkgVersion,
                 runsDir: settings.runsDir,
                 mcpServerAuthHeaders,
@@ -762,97 +909,6 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                     message: `Evaluation execution finished (run id: ${results.metadata.run_id})`
                 }
             });
-            if (applySnapshotEval && expandedConfig.snapshot_eval?.enabled) {
-                addJobEvent(job, {
-                    type: 'log',
-                    ts: new Date().toISOString(),
-                    payload: { message: 'Applying snapshot evaluation policy ...' }
-                });
-                const policy = expandedConfig.snapshot_eval;
-                const enabledScenarioIds = new Set(selectedBaseScenarios.scenarios
-                    .filter((scenario) => scenario.snapshot_eval?.enabled !== false)
-                    .map((scenario) => scenario.id));
-                const scenarioBaselineMap = new Map();
-                for (const scenario of selectedBaseScenarios.scenarios) {
-                    if (scenario.snapshot_eval?.enabled === false)
-                        continue;
-                    const baselineId = scenario.snapshot_eval?.baseline_snapshot_id ?? policy.baseline_snapshot_id;
-                    if (baselineId)
-                        scenarioBaselineMap.set(scenario.id, baselineId);
-                }
-                const scenariosWithoutBaseline = selectedBaseScenarios.scenarios
-                    .filter((scenario) => scenario.snapshot_eval?.enabled !== false)
-                    .filter((scenario) => !(scenario.snapshot_eval?.baseline_snapshot_id ?? policy.baseline_snapshot_id))
-                    .map((scenario) => scenario.id);
-                if (scenariosWithoutBaseline.length > 0) {
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: `Snapshot eval enabled but no baseline configured for scenarios: ${scenariosWithoutBaseline.join(', ')}`
-                        }
-                    });
-                }
-                const comparisons = [];
-                const scenarioIdsByBaseline = new Map();
-                for (const [scenarioIdItem, baselineId] of scenarioBaselineMap) {
-                    const list = scenarioIdsByBaseline.get(baselineId) ?? [];
-                    list.push(scenarioIdItem);
-                    scenarioIdsByBaseline.set(baselineId, list);
-                }
-                for (const [baselineId, scenarioIdsForBaseline] of scenarioIdsByBaseline) {
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: `Comparing ${scenarioIdsForBaseline.length} scenario(s) to snapshot baseline '${baselineId}'`
-                        }
-                    });
-                    const snapshot = loadSnapshot(baselineId, settings.snapshotsDir);
-                    const fullComparison = compareRunToSnapshot(results, snapshot);
-                    comparisons.push({
-                        ...fullComparison,
-                        scenario_results: fullComparison.scenario_results.filter((row) => scenarioIdsForBaseline.includes(row.scenario_id))
-                    });
-                }
-                if (comparisons.length > 0) {
-                    applySnapshotPolicyToRunResult({ results, comparisons, policy, enabledScenarioIds });
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: `Snapshot evaluation applied (${comparisons.length} baseline comparison group(s))`
-                        }
-                    });
-                }
-                else {
-                    addJobEvent(job, {
-                        type: 'log',
-                        ts: new Date().toISOString(),
-                        payload: {
-                            message: 'Snapshot evaluation enabled, but no baseline comparisons were applied'
-                        }
-                    });
-                }
-            }
-            else if (applySnapshotEval) {
-                addJobEvent(job, {
-                    type: 'log',
-                    ts: new Date().toISOString(),
-                    payload: {
-                        message: 'Snapshot evaluation requested, but config snapshot evaluation is disabled'
-                    }
-                });
-            }
-            else {
-                addJobEvent(job, {
-                    type: 'log',
-                    ts: new Date().toISOString(),
-                    payload: {
-                        message: 'Snapshot evaluation skipped for this run (disabled in run request)'
-                    }
-                });
-            }
             addJobEvent(job, {
                 type: 'log',
                 ts: new Date().toISOString(),
@@ -874,8 +930,7 @@ async function executeRunJob(job, settings, jobs, runQueueState, oauthSessionMan
                 payload: {
                     runId: results.metadata.run_id,
                     runDir,
-                    summary: results.summary,
-                    snapshotEval: results.metadata.snapshot_eval ?? null
+                    summary: results.summary
                 }
             });
             job.status = 'completed';
@@ -933,9 +988,9 @@ function formatRunProgressMessage(event) {
         case 'run_started':
             return `Run initialized (id: ${event.runId}, ${event.totalScenarioRuns} scenario run(s))`;
         case 'mcp_connect_started':
-            return `Connecting to ${event.serverCount} MCP server(s) ...`;
+            return `Connecting to ${event.serverCount} MCP server(s): ${event.serverNames.join(', ')} ...`;
         case 'mcp_connect_finished':
-            return `Connected to ${event.serverCount} MCP server(s)`;
+            return `Connected to ${event.serverCount} MCP server(s): ${event.serverNames.join(', ')}`;
         case 'scenario_run_started':
             return `Scenario ${event.scenarioRunIndex}/${event.totalScenarioRuns} started: ${event.scenarioId} [agent=${event.agentName}, run=${event.runIndex + 1}/${event.runsPerScenario}]`;
         case 'scenario_run_finished':