@inspectr/mcplab-mcp-server 1.8.3 → 1.9.0

package/dist/runtime.js

@@ -5,7 +5,7 @@ import { basename, dirname, extname, resolve, join, sep, relative } from 'node:p
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
-import { loadConfig, hashConfig, runAll, selectScenarios, loadOrBuildSearchIndex, indexNeedsRefresh, getResultsIndexPaths, resolveRunArtifactPath, searchDocs, getContext, applyRuntimeServerOverrides, readLibraryAgentsAndServers, resolveScenarioLibraryDir, createEvaluationConfigFile, updateEvaluationConfigFile, readEvaluationConfigFile, createTestCaseFile, readTestCaseFile, updateTestCaseFile, buildScenarioEntry } from '@inspectr/mcplab-core';
+import { loadConfig, hashConfig, runAll, selectScenarios, loadOrBuildSearchIndex, indexNeedsRefresh, getResultsIndexPaths, resolveRunArtifactPath, searchDocs, getContext, applyRuntimeServerOverrides, readLibraryAgentsAndServers, readBrowserProviderProfiles, serializeBrowserProviderProfile, resolveScenarioLibraryDir, createEvaluationConfigFile, updateEvaluationConfigFile, readEvaluationConfigFile, createTestCaseFile, readTestCaseFile, updateTestCaseFile, buildScenarioEntry, DEFAULT_BROWSER_PROVIDER_PROFILES, mergeBrowserProviderProfiles } from '@inspectr/mcplab-core';
 import { renderReport } from '@inspectr/mcplab-reporting';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
 import { z } from 'zod';
@@ -25,7 +25,7 @@ const GenericObjectSchema = z.object({}).passthrough();
 const ResultsSummarySchema = z.object({
     total_scenarios: z.number().int().nonnegative(),
     total_runs: z.number().int().nonnegative(),
-    pass_rate: z.number(),
+    pass_rate: z.number().nullable(),
     avg_tool_calls_per_run: z.number(),
     avg_tool_latency_ms: z.number().nullable()
 });
@@ -44,7 +44,8 @@ const MetricSummarySchema = z.object({
     total_runs: z.number().int().nonnegative(),
     passed_runs: z.number().int().nonnegative(),
     failed_runs: z.number().int().nonnegative(),
-    pass_rate: z.number(),
+    stopped_runs: z.number().int().nonnegative().optional(),
+    pass_rate: z.number().nullable(),
     avg_tool_calls_per_run: z.number(),
     avg_tool_latency_ms: z.number().nullable()
 });
@@ -66,7 +67,7 @@ const CompareRowSchema = z.object({
     key: z.string(),
     scenario_id: z.string(),
     agent: z.string(),
-    classification: z.enum(['regressed', 'improved', 'unchanged', 'new', 'missing']),
+    classification: z.enum(['regressed', 'improved', 'unchanged', 'new', 'missing', 'not_evaluated']),
     left: MetricSummarySchema.nullable(),
     right: MetricSummarySchema.nullable(),
     deltas: z.object({
@@ -96,6 +97,13 @@ const AgentEntrySchema = z.object({
     max_tokens: z.number().int().positive().optional(),
     system: z.string().optional()
 });
+const BrowserAgentEntrySchema = z.object({
+    type: z.literal('browser'),
+    name: z.string().optional(),
+    provider: z.string(),
+    url: z.string(),
+    new_conversation_between_scenarios: z.boolean().optional()
+});
 const LibraryServerEntryContentSchema = z
     .object({
     transport: z.string().optional(),
@@ -112,12 +120,17 @@ const LibraryServerEntrySchema = z.object({
 });
 const LibraryAgentEntrySchema = z.object({
     id: z.string(),
-    entry: AgentEntrySchema.optional()
+    entry: z.union([AgentEntrySchema, BrowserAgentEntrySchema]).optional()
+});
+const LibraryBrowserProviderEntrySchema = z.object({
+    id: z.string(),
+    entry: GenericObjectSchema.optional()
 });
 const LibraryEntrySchema = z.object({
     bundleRoot: z.string(),
     servers: z.array(LibraryServerEntrySchema),
     agents: z.array(LibraryAgentEntrySchema),
+    browser_providers: z.array(LibraryBrowserProviderEntrySchema),
     test_cases: z.array(LibraryScenarioEntrySchema),
     scenarios: z.array(LibraryScenarioEntrySchema)
 });
@@ -615,7 +628,7 @@ export async function startMcplabMcpServer(options) {
     const sessions = new Map();
     const httpServer = createServer(async (req, res) => {
         try {
-            await handleHttpRequest(req, res, sessions, options.path);
+            await handleHttpRequest(req, res, sessions, options.path, logger);
         }
         catch (error) {
             logger.error('[mcplab-mcp] request error:', error);
@@ -1081,7 +1094,7 @@ export function registerTools(server) {
         outputSchema: LibraryEntrySchema,
         inputSchema: {
             kind: z
-                .enum(['all', 'servers', 'agents', 'test_cases', 'scenarios'])
+                .enum(['all', 'servers', 'agents', 'browser_providers', 'test_cases', 'scenarios'])
                 .optional()
                 .describe('Which library category to list. Defaults to all.'),
             includeContent: z
@@ -1091,15 +1104,16 @@ export function registerTools(server) {
         }
     }, async ({ kind, includeContent }) => {
         return withToolHandling(async () => {
-            const root = resolveBundleRoot();
-            const data = readLibrary(root, Boolean(includeContent));
             const selectedKind = kind ?? 'all';
+            const root = resolveBundleRoot();
+            const data = readLibrary(root, Boolean(includeContent), selectedKind);
             const structured = selectedKind === 'all'
                 ? data
                 : {
                     bundleRoot: data.bundleRoot,
                     servers: selectedKind === 'servers' ? data.servers : [],
                     agents: selectedKind === 'agents' ? data.agents : [],
+                    browser_providers: selectedKind === 'browser_providers' ? data.browser_providers : [],
                     test_cases: selectedKind === 'test_cases' ? data.test_cases : [],
                     scenarios: selectedKind === 'scenarios' ? data.scenarios : []
                 };
@@ -1107,10 +1121,10 @@ export function registerTools(server) {
         });
     });
     registerTool('mcplab_get_library_item', {
-        description: 'Get a specific reusable server, agent, or scenario definition from a MCPLab library bundle and return both structured data and YAML.',
+        description: 'Get a specific reusable server, agent, browser provider, test case, or scenario definition from a MCPLab library bundle and return both structured data and YAML.',
         outputSchema: {
             bundleRoot: z.string(),
-            kind: z.enum(['servers', 'agents', 'test_cases', 'scenarios']),
+            kind: z.enum(['servers', 'agents', 'browser_providers', 'test_cases', 'scenarios']),
             id: z.string(),
             file: z.string().optional(),
             yaml: z.string(),
@@ -1118,7 +1132,7 @@ export function registerTools(server) {
         },
         inputSchema: {
             kind: z
-                .enum(['servers', 'agents', 'test_cases', 'scenarios'])
+                .enum(['servers', 'agents', 'browser_providers', 'test_cases', 'scenarios'])
                 .describe('Library category.'),
             id: z.string().describe('Entry id (for scenarios this is scenario.id, not filename).')
         }
@@ -1175,6 +1189,46 @@ export function registerTools(server) {
             });
         });
     });
+    registerTool('mcplab_generate_browser_agent_entry', {
+        description: 'Generate an agents.yaml entry for a Rover browser agent. Browser agents reference a validated browser provider profile and are executed by Rover, not by an LLM API.',
+        outputSchema: {
+            id: z.string(),
+            entry: z.object({
+                type: z.literal('browser'),
+                name: z.string().optional(),
+                provider: z.string(),
+                url: z.string(),
+                new_conversation_between_scenarios: z.boolean().optional()
+            }),
+            yaml: z.string()
+        },
+        inputSchema: {
+            id: z.string().describe('Browser agent id key (kebab-case recommended).'),
+            name: z.string().optional().describe('Human-readable browser agent name.'),
+            provider: z
+                .string()
+                .min(1)
+                .describe('Provider id from browser-providers/<provider-id>.yaml.'),
+            url: z.string().url().describe('Browser origin where Rover should execute the agent.'),
+            new_conversation_between_scenarios: z
+                .boolean()
+                .optional()
+                .describe('Whether Rover should start a new conversation between scenarios.')
+        }
+    }, async ({ id, name, provider, url, new_conversation_between_scenarios }) => withToolHandling(async () => {
+        const entry = removeUndefined({
+            type: 'browser',
+            name,
+            provider,
+            url,
+            new_conversation_between_scenarios
+        });
+        return ok(`Generated browser agent entry '${id}'`, {
+            id,
+            entry,
+            yaml: stringifyYaml({ [id]: entry }).trimEnd()
+        });
+    }));
     registerTool('mcplab_create_test_case', {
         description: 'Create one reviewed MCPLab Test Case in mcplab/test-cases/. Put evaluation checks in the eval object for full support of response, tool-input, and agent assertions. Simple shorthand fields are supported for basic cases; do not provide a shorthand field together with its equivalent eval field. This is a scoped YAML write and refuses duplicate IDs.',
         outputSchema: { id: z.string(), path: z.string(), test_case: ScenarioEntrySchema },
@@ -1556,7 +1610,7 @@ export function registerTools(server) {
             scenarios: z.array(z.object({
                 scenario_id: z.string(),
                 agent: z.string(),
-                pass_rate: z.number(),
+                pass_rate: z.number().nullable(),
                 tool_usage_frequency: z.record(z.number())
             })),
             effective_server_overrides: z.record(z.array(z.string())),
@@ -1663,7 +1717,7 @@ export function registerTools(server) {
             throw new Error('since must be earlier than or equal to until');
         const matching = listRunsWithFallback(resolveRunsDir(), undefined, true).filter((entry) => {
             const timestamp = String(entry.metadata && typeof entry.metadata === 'object'
-                ? entry.metadata.timestamp ?? ''
+                ? (entry.metadata.timestamp ?? '')
                 : '');
             const time = new Date(timestamp).getTime();
             if (sinceMs === undefined && untilMs === undefined)
@@ -1857,7 +1911,7 @@ export function registerTools(server) {
                 left: MetricSummarySchema,
                 right: MetricSummarySchema,
                 deltas: z.object({
-                    pass_rate: z.number(),
+                    pass_rate: z.number().nullable(),
                     failed_runs: z.number(),
                     avg_tool_calls_per_run: z.number(),
                     avg_tool_latency_ms: z.number().nullable()
@@ -1867,7 +1921,8 @@ export function registerTools(server) {
                     improved: z.number().int().nonnegative(),
                     unchanged: z.number().int().nonnegative(),
                     new: z.number().int().nonnegative(),
-                    missing: z.number().int().nonnegative()
+                    missing: z.number().int().nonnegative(),
+                    not_evaluated: z.number().int().nonnegative()
                 })
             }),
             regressions: z.array(CompareRowSchema),
@@ -2389,7 +2444,7 @@ export function registerTools(server) {
                 .array(z.object({
                 scenario_id: z.string(),
                 agent: z.string(),
-                pass_rate: z.number()
+                pass_rate: z.number().nullable()
             }))
                 .optional()
         },
@@ -2805,15 +2860,18 @@ function mergeLibraryServersAndAgents(config, bundleRoot) {
         servers: { ...libraries.servers, ...config.servers }
     };
 }
-function readLibrary(bundleRoot, includeContent) {
+function readLibrary(bundleRoot, includeContent, kind = 'all') {
     const serversPath = join(bundleRoot, 'servers.yaml');
     const agentsPath = join(bundleRoot, 'agents.yaml');
     const scenariosDir = resolveScenarioLibraryDir(bundleRoot).path;
     const servers = existsSync(serversPath)
-        ? parseYaml(readFileSync(serversPath, 'utf8')) ?? {}
+        ? (parseYaml(readFileSync(serversPath, 'utf8')) ?? {})
         : {};
     const agents = existsSync(agentsPath)
-        ? parseYaml(readFileSync(agentsPath, 'utf8')) ?? {}
+        ? (parseYaml(readFileSync(agentsPath, 'utf8')) ?? {})
+        : {};
+    const browserProviders = kind === 'all' || kind === 'browser_providers'
+        ? mergeBrowserProviderProfiles(readBrowserProviderProfiles(bundleRoot))
         : {};
     const scenarioEntries = [];
     if (existsSync(scenariosDir)) {
@@ -2845,29 +2903,57 @@ function readLibrary(bundleRoot, includeContent) {
             id,
             ...(includeContent ? { entry: agents[id] } : {})
         })),
+        browser_providers: Object.keys(browserProviders)
+            .sort()
+            .map((id) => ({
+            id,
+            ...(includeContent
+                ? { entry: browserProviders[id] }
+                : {})
+        })),
         test_cases: scenarioEntries,
         scenarios: scenarioEntries
     };
     return out;
 }
 function getLibraryItem(bundleRoot, kind, id) {
-    if (kind === 'servers' || kind === 'agents') {
-        const file = join(bundleRoot, `${kind}.yaml`);
+    if (kind === 'servers' || kind === 'agents' || kind === 'browser_providers') {
+        const directory = kind === 'browser_providers' ? join(bundleRoot, 'browser-providers') : null;
+        const builtIn = kind === 'browser_providers' && Object.hasOwn(DEFAULT_BROWSER_PROVIDER_PROFILES, id)
+            ? DEFAULT_BROWSER_PROVIDER_PROFILES[id]
+            : undefined;
+        const file = directory
+            ? findBrowserProviderFile(directory, id)
+            : join(bundleRoot, `${kind}.yaml`);
+        if (builtIn && !existsSync(file)) {
+            const content = structuredClone(builtIn);
+            return {
+                bundleRoot,
+                kind,
+                id,
+                yaml: serializeBrowserProviderProfile(content).trimEnd(),
+                content: content
+            };
+        }
         if (!existsSync(file)) {
             throw new Error(`Library file not found: ${file}`);
         }
         const raw = readFileSync(file, 'utf8');
         const parsed = parseYaml(raw) ?? {};
-        if (!(id in parsed)) {
+        if (kind !== 'browser_providers' && !(id in parsed)) {
             throw new Error(`'${id}' not found in ${file}`);
         }
-        const entry = parsed[id];
+        const normalizedEntry = kind === 'browser_providers' ? readBrowserProviderProfiles(bundleRoot)[id] : parsed[id];
+        if (!normalizedEntry)
+            throw new Error(`'${id}' not found in ${file}`);
         return {
             bundleRoot,
             kind,
             id,
-            yaml: stringifyYaml({ [id]: entry }).trimEnd(),
-            content: entry
+            yaml: kind === 'browser_providers'
+                ? raw.trimEnd()
+                : stringifyYaml({ [id]: normalizedEntry }).trimEnd(),
+            content: normalizedEntry
         };
     }
     const dir = resolveScenarioLibraryDir(bundleRoot).path;
@@ -2892,6 +2978,14 @@ function getLibraryItem(bundleRoot, kind, id) {
     }
     throw new Error(`Scenario '${id}' not found in ${dir}`);
 }
+function findBrowserProviderFile(directory, id) {
+    if (!existsSync(directory))
+        return join(directory, `${id}.yaml`);
+    const file = readdirSync(directory)
+        .filter((name) => name.endsWith('.yaml') || name.endsWith('.yml'))
+        .find((name) => basename(name, extname(name)) === id);
+    return join(directory, file ?? `${id}.yaml`);
+}
 function listEvaluationConfigs(params) {
     const evalsDir = join(resolveBundleRoot(), 'evals');
     const files = [];
@@ -3762,9 +3856,9 @@ function truncate(text, maxChars) {
     return `${text.slice(0, maxChars)}\n...[truncated ${text.length - maxChars} chars]`;
 }
 export async function handleMcplabMcpHttpRequest(req, res, sessions, options) {
-    await handleHttpRequest(req, res, sessions, options?.path ?? DEFAULT_MCP_PATH);
+    await handleHttpRequest(req, res, sessions, options?.path ?? DEFAULT_MCP_PATH, options?.logger ?? console);
 }
-async function handleHttpRequest(req, res, sessions, mcpPath) {
+async function handleHttpRequest(req, res, sessions, mcpPath, logger = console) {
     const method = req.method ?? 'GET';
     const pathname = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`).pathname;
     if (pathname === '/' && method === 'GET') {
@@ -3796,14 +3890,25 @@ async function handleHttpRequest(req, res, sessions, mcpPath) {
             return;
         }
         if (isInitializeRequest(body)) {
+            const clientInfo = body.params && typeof body.params === 'object' && 'clientInfo' in body.params
+                ? body.params.clientInfo
+                : undefined;
+            const clientName = clientInfo && typeof clientInfo === 'object' && 'name' in clientInfo
+                ? String(clientInfo.name)
+                : 'unknown';
+            const clientVersion = clientInfo && typeof clientInfo === 'object' && 'version' in clientInfo
+                ? String(clientInfo.version)
+                : undefined;
             let runtime;
             const transport = new StreamableHTTPServerTransport({
                 sessionIdGenerator: () => randomUUID(),
                 onsessioninitialized: (sid) => {
                     sessions.set(sid, runtime);
+                    logger.log(`[mcplab-mcp] client connected: ${clientName}${clientVersion ? `/${clientVersion}` : ''} (session ${sid})`);
                 },
                 onsessionclosed: (sid) => {
                     sessions.delete(sid);
+                    logger.log(`[mcplab-mcp] client disconnected: session ${sid}`);
                 }
             });
             const mcpServer = createConfiguredServer();