@inspectr/mcplab-core 0.1.0

package/dist/eval.js

@@ -0,0 +1,111 @@
+import { JSONPath } from 'jsonpath-plus';
+export function evaluateScenario(finalText, toolSequence, evalRules) {
+    const failures = [];
+    if (evalRules?.tool_constraints) {
+        failures.push(...evaluateToolConstraints(toolSequence, evalRules.tool_constraints));
+    }
+    if (evalRules?.tool_sequence?.allow?.length) {
+        failures.push(...evaluateToolSequence(toolSequence, evalRules.tool_sequence.allow));
+    }
+    if (evalRules?.response_assertions?.length) {
+        failures.push(...evaluateResponseAssertions(finalText, evalRules.response_assertions));
+    }
+    return { pass: failures.length === 0, failures };
+}
+function evaluateToolConstraints(toolSequence, constraints) {
+    const failures = [];
+    const unique = new Set(toolSequence);
+    if (constraints.forbidden_tools) {
+        for (const tool of constraints.forbidden_tools) {
+            if (unique.has(tool)) {
+                failures.push(`Forbidden tool used: ${tool}`);
+            }
+        }
+    }
+    if (constraints.required_tools) {
+        for (const tool of constraints.required_tools) {
+            if (!unique.has(tool)) {
+                failures.push(`Required tool not used: ${tool}`);
+            }
+        }
+    }
+    return failures;
+}
+function evaluateToolSequence(actual, allowed) {
+    const actualKey = JSON.stringify(actual);
+    const allowedKeys = new Set(allowed.map((seq) => JSON.stringify(seq)));
+    if (!allowedKeys.has(actualKey)) {
+        return ['Tool sequence did not match any allowed sequence'];
+    }
+    return [];
+}
+function evaluateResponseAssertions(text, assertions) {
+    const failures = [];
+    for (const assertion of assertions) {
+        if (assertion.type === 'regex') {
+            try {
+                // Default text pattern checks to case-insensitive to reduce brittle LLM-output casing failures.
+                const re = new RegExp(assertion.pattern, 'i');
+                if (!re.test(text)) {
+                    failures.push(`Regex assertion failed: ${assertion.pattern}`);
+                }
+            }
+            catch (err) {
+                failures.push(`Invalid regex: ${assertion.pattern}`);
+            }
+        }
+        if (assertion.type === 'jsonpath') {
+            let json;
+            try {
+                json = JSON.parse(text);
+            }
+            catch {
+                failures.push(`JSONPath assertion failed: invalid JSON for path ${assertion.path}`);
+                continue;
+            }
+            const result = JSONPath({ path: assertion.path, json });
+            if (assertion.equals !== undefined) {
+                const matched = result.some((value) => value === assertion.equals);
+                if (!matched) {
+                    failures.push(`JSONPath equals assertion failed: ${assertion.path}`);
+                }
+            }
+            else if (!result || result.length === 0) {
+                failures.push(`JSONPath assertion failed: ${assertion.path}`);
+            }
+        }
+    }
+    return failures;
+}
+export function extractValues(text, extractRules = []) {
+    const extracted = {};
+    for (const rule of extractRules) {
+        try {
+            const re = new RegExp(rule.regex);
+            const match = re.exec(text);
+            if (!match) {
+                extracted[rule.name] = null;
+                continue;
+            }
+            const value = match.groups?.value ?? match[1] ?? match[0];
+            extracted[rule.name] = coerceValue(value);
+        }
+        catch {
+            extracted[rule.name] = null;
+        }
+    }
+    return extracted;
+}
+function coerceValue(value) {
+    const trimmed = value.trim();
+    if (trimmed === 'true')
+        return true;
+    if (trimmed === 'false')
+        return false;
+    const num = Number(trimmed);
+    if (!Number.isNaN(num) && trimmed !== '') {
+        return num;
+    }
+    return value;
+}
+//# sourceMappingURL=eval.js.map

package/dist/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAQzC,MAAM,UAAU,gBAAgB,CAC9B,SAAiB,EACjB,YAAsB,EACtB,SAAqB;IAErB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,SAAS,EAAE,gBAAgB,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC,GAAG,uBAAuB,CAAC,YAAY,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,YAAY,EAAE,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAC3C,QAAQ,CAAC,IAAI,CAAC,GAAG,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACzF,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAsB,EAAE,WAA4B;IACnF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;IACrC,IAAI,WAAW,CAAC,eAAe,EAAE,CAAC;QAChC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,eAAe,EAAE,CAAC;YAC/C,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,cAAc,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,cAAc,EAAE,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAgB,EAAE,OAAmB;IACjE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,kDAAkD,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,0BAA0B,CAAC,IAAY,EAAE,UAA+B;IAC/E,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,gGAAgG;gBAChG,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC9C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnB,QAAQ,CAAC,IAAI,CAAC,2BAA2B,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,QAAQ,CAAC,IAAI,CAAC,kBAAkB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAClC,IAAI,IAAS,CAAC;YACd,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,oDAAoD,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACxD,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,QAAQ,CAAC,IAAI,CAAC,qCAAqC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1C,QAAQ,CAAC,IAAI,CAAC,8BAA8B,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,eAAkD,EAAE;IAEpD,MAAM,SAAS,GAAqD,EAAE,CAAC;IACvE,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;gBAC5B,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1D,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACzC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export * from './types.js';
+export { loadConfig, selectScenarios, expandConfigForAgents, normalizeSourceConfig, normalizeLibraryServers, normalizeLibraryAgents } from './config.js';
+export { runAll, type RunProgressEvent } from './runner.js';
+export { McpClientManager } from './mcp.js';
+export { chatWithAgent } from './agent.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EACL,UAAU,EACV,eAAe,EACf,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,sBAAsB,EACvB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,MAAM,EAAE,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,6 @@
+export * from './types.js';
+export { loadConfig, selectScenarios, expandConfigForAgents, normalizeSourceConfig, normalizeLibraryServers, normalizeLibraryAgents } from './config.js';
+export { runAll } from './runner.js';
+export { McpClientManager } from './mcp.js';
+export { chatWithAgent } from './agent.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EACL,UAAU,EACV,eAAe,EACf,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,sBAAsB,EACvB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,MAAM,EAAyB,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/mcp.d.ts

@@ -0,0 +1,19 @@
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import type { ServerConfig, ToolDef } from './types.js';
+export interface McpToolInfo {
+  name: string;
+  description?: string;
+  inputSchema?: unknown;
+}
+export declare class McpClientManager {
+  private clients;
+  private oauthCache;
+  private static readonly MAX_CONNECT_RETRIES;
+  connectAll(servers: Record<string, ServerConfig>, signal?: AbortSignal): Promise<void>;
+  getClient(serverName: string): Client;
+  listTools(serverName: string): Promise<ToolDef[]>;
+  callTool(serverName: string, tool: string, args: unknown): Promise<any>;
+  private getAuthHeaders;
+  private fetchOauthToken;
+}
+//# sourceMappingURL=mcp.d.ts.map

package/dist/mcp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAEnE,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,UAAU,CAA2D;IAC7E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAK;IAE1C,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAyC5F,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAQ/B,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAqBjD,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;YAe/D,cAAc;YAoCd,eAAe;CAyD9B"}

package/dist/mcp.js

@@ -0,0 +1,198 @@
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+export class McpClientManager {
+    clients = new Map();
+    oauthCache = new Map();
+    static MAX_CONNECT_RETRIES = 3;
+    async connectAll(servers, signal) {
+        throwIfAborted(signal);
+        for (const [name, server] of Object.entries(servers)) {
+            throwIfAborted(signal);
+            if (server.transport !== 'http') {
+                throw new Error(`Unsupported transport for server ${name}: ${server.transport}`);
+            }
+            let lastError;
+            for (let attempt = 0; attempt <= McpClientManager.MAX_CONNECT_RETRIES; attempt += 1) {
+                try {
+                    const client = new Client({
+                        name: `mcp-eval-${name}`,
+                        version: '0.1.0'
+                    });
+                    const headers = await this.getAuthHeaders(name, server);
+                    const transport = new StreamableHTTPClientTransport(new URL(server.url), {
+                        requestInit: { headers }
+                    });
+                    await client.connect(transport);
+                    this.clients.set(name, client);
+                    lastError = undefined;
+                    break;
+                }
+                catch (err) {
+                    throwIfAborted(signal);
+                    lastError = err;
+                    if (attempt >= McpClientManager.MAX_CONNECT_RETRIES)
+                        break;
+                    await sleep(250 * (attempt + 1), signal);
+                }
+            }
+            if (lastError) {
+                throw new Error(formatMcpError(`Failed to connect to MCP server '${name}' after ${McpClientManager.MAX_CONNECT_RETRIES} retries`, server.url, lastError));
+            }
+        }
+    }
+    getClient(serverName) {
+        const client = this.clients.get(serverName);
+        if (!client) {
+            throw new Error(`MCP client not found for server: ${serverName}`);
+        }
+        return client;
+    }
+    async listTools(serverName) {
+        const client = this.getClient(serverName);
+        try {
+            const result = await client.listTools();
+            const tools = Array.isArray(result?.tools)
+                ? result.tools
+                : Array.isArray(result)
+                    ? result
+                    : [];
+            return tools.map((tool) => ({
+                name: tool.name,
+                description: tool.description,
+                inputSchema: tool.inputSchema ?? tool.input_schema ?? tool.input
+            }));
+        }
+        catch (err) {
+            throw new Error(formatMcpError(`Failed to list tools for server '${serverName}'`, undefined, err));
+        }
+    }
+    async callTool(serverName, tool, args) {
+        const client = this.getClient(serverName);
+        try {
+            const result = await client.callTool({
+                name: tool,
+                arguments: args
+            });
+            return result;
+        }
+        catch (err) {
+            throw new Error(formatMcpError(`Tool call failed '${tool}' on server '${serverName}'`, undefined, err));
+        }
+    }
+    async getAuthHeaders(serverName, server) {
+        const headers = {};
+        if (!server.auth)
+            return headers;
+        if (server.auth.type === 'bearer') {
+            const token = process.env[server.auth.env];
+            if (!token) {
+                throw new Error(`Missing bearer token env var: ${server.auth.env}`);
+            }
+            headers['Authorization'] = `Bearer ${token}`;
+            return headers;
+        }
+        if (server.auth.type === 'oauth_client_credentials') {
+            const cached = this.oauthCache.get(serverName);
+            if (cached && cached.expiresAt > Date.now() + 30_000) {
+                headers['Authorization'] = `Bearer ${cached.token}`;
+                return headers;
+            }
+            const token = await this.fetchOauthToken(serverName, server);
+            headers['Authorization'] = `Bearer ${token}`;
+            return headers;
+        }
+        if (server.auth.type === 'oauth_authorization_code') {
+            throw new Error(`Server '${serverName}' uses OAuth authorization_code metadata. This auth type is intended for OAuth Debugger setup, not automated MCP runtime connections. Use bearer or oauth_client_credentials for runs.`);
+        }
+        return headers;
+    }
+    async fetchOauthToken(serverName, server) {
+        if (!server.auth || server.auth.type !== 'oauth_client_credentials') {
+            throw new Error(`OAuth auth not configured for server '${serverName}'`);
+        }
+        const clientId = process.env[server.auth.client_id_env];
+        const clientSecret = process.env[server.auth.client_secret_env];
+        if (!clientId) {
+            throw new Error(`Missing OAuth client id env var: ${server.auth.client_id_env}`);
+        }
+        if (!clientSecret) {
+            throw new Error(`Missing OAuth client secret env var: ${server.auth.client_secret_env}`);
+        }
+        const params = new URLSearchParams();
+        params.set('grant_type', 'client_credentials');
+        params.set('client_id', clientId);
+        params.set('client_secret', clientSecret);
+        if (server.auth.scope)
+            params.set('scope', server.auth.scope);
+        if (server.auth.audience)
+            params.set('audience', server.auth.audience);
+        if (server.auth.token_params) {
+            for (const [key, value] of Object.entries(server.auth.token_params)) {
+                params.set(key, value);
+            }
+        }
+        try {
+            const response = await fetch(server.auth.token_url, {
+                method: 'POST',
+                headers: { 'content-type': 'application/x-www-form-urlencoded' },
+                body: params.toString()
+            });
+            if (!response.ok) {
+                const text = await safeReadText(response);
+                throw new Error(`Token request failed: ${response.status} ${response.statusText} ${text}`);
+            }
+            const data = (await response.json());
+            const accessToken = data.access_token;
+            if (!accessToken) {
+                throw new Error('Token response missing access_token');
+            }
+            const ttl = typeof data.expires_in === 'number' ? data.expires_in * 1000 : 60 * 60 * 1000;
+            this.oauthCache.set(serverName, { token: accessToken, expiresAt: Date.now() + ttl });
+            return accessToken;
+        }
+        catch (err) {
+            throw new Error(formatMcpError(`Failed to fetch OAuth token for server '${serverName}'`, server.auth.token_url, err));
+        }
+    }
+}
+function formatMcpError(prefix, url, err) {
+    const message = err?.message ?? String(err);
+    const hints = [];
+    if (message.includes('fetch failed')) {
+        hints.push('Verify the MCP server is running and reachable.');
+        if (url)
+            hints.push(`Check the URL: ${url}`);
+        hints.push('If auth is required, confirm the bearer token env var is set.');
+    }
+    const hintText = hints.length > 0 ? ` Hints: ${hints.join(' ')}` : '';
+    return `${prefix}. ${message}.${hintText}`;
+}
+async function safeReadText(response) {
+    try {
+        return await response.text();
+    }
+    catch {
+        return '';
+    }
+}
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(new Error('Run aborted by user'));
+        };
+        if (signal) {
+            signal.addEventListener('abort', onAbort, { once: true });
+        }
+    });
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw new Error('Run aborted by user');
+    }
+}
+//# sourceMappingURL=mcp.js.map

package/dist/mcp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AASnG,MAAM,OAAO,gBAAgB;IACnB,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,UAAU,GAAG,IAAI,GAAG,EAAgD,CAAC;IACrE,MAAM,CAAU,mBAAmB,GAAG,CAAC,CAAC;IAEhD,KAAK,CAAC,UAAU,CAAC,OAAqC,EAAE,MAAoB;QAC1E,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,cAAc,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,MAAM,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,SAAc,CAAC;YACnB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,gBAAgB,CAAC,mBAAmB,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;gBACpF,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;wBACxB,IAAI,EAAE,YAAY,IAAI,EAAE;wBACxB,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;oBACxD,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvE,WAAW,EAAE,EAAE,OAAO,EAAE;qBACzB,CAAC,CAAC;oBACH,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;oBAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;oBAC/B,SAAS,GAAG,SAAS,CAAC;oBACtB,MAAM;gBACR,CAAC;gBAAC,OAAO,GAAQ,EAAE,CAAC;oBAClB,cAAc,CAAC,MAAM,CAAC,CAAC;oBACvB,SAAS,GAAG,GAAG,CAAC;oBAChB,IAAI,OAAO,IAAI,gBAAgB,CAAC,mBAAmB;wBAAE,MAAM;oBAC3D,MAAM,KAAK,CAAC,GAAG,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CACb,cAAc,CACZ,oCAAoC,IAAI,WAAW,gBAAgB,CAAC,mBAAmB,UAAU,EACjG,MAAM,CAAC,GAAG,EACV,SAAS,CACV,CACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAS,CAAC,UAAkB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,oCAAoC,UAAU,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,UAAkB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,MAAM,GAAQ,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC;gBACxC,CAAC,CAAC,MAAM,CAAC,KAAK;gBACd,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;oBACvB,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK;aACjE,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,cAAc,CAAC,oCAAoC,UAAU,GAAG,EAAE,SAAS,EAAE,GAAG,CAAC,CAClF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,IAAa;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBACnC,IAAI,EAAE,IAAI;gBACV,SAAS,EAAE,IAAW;aACvB,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,cAAc,CAAC,qBAAqB,IAAI,gBAAgB,UAAU,GAAG,EAAE,SAAS,EAAE,GAAG,CAAC,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,UAAkB,EAClB,MAAoB;QAEpB,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,OAAO,OAAO,CAAC;QAEjC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACtE,CAAC;YACD,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,KAAK,EAAE,CAAC;YAC7C,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,0BAA0B,EAAE,CAAC;YACpD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC/C,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC;gBACrD,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC;gBACpD,OAAO,OAAO,CAAC;YACjB,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC7D,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,KAAK,EAAE,CAAC;YAC7C,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,0BAA0B,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,wLAAwL,CAC9M,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,UAAkB,EAAE,MAAoB;QACpE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,0BAA0B,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,UAAU,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACxD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oCAAoC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvE,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;gBAClD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,mCAAmC,EAAE;gBAChE,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE;aACxB,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAC1C,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC,CAAC;YAC7F,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC;YACtC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACzD,CAAC;YACD,MAAM,GAAG,GAAG,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;YAC1F,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YACrF,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,cAAc,CACZ,2CAA2C,UAAU,GAAG,EACxD,MAAM,CAAC,IAAI,CAAC,SAAS,EACrB,GAAG,CACJ,CACF,CAAC;QACJ,CAAC;IACH,CAAC;;AAGH,SAAS,cAAc,CAAC,MAAc,EAAE,GAAuB,EAAE,GAAQ;IACvE,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAC9D,IAAI,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtE,OAAO,GAAG,MAAM,KAAK,OAAO,IAAI,QAAQ,EAAE,CAAC;AAC7C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAkB;IAC5C,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAAoB;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC;QACF,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC"}

package/dist/results.d.ts

@@ -0,0 +1,17 @@
+import type { EvalRules, ResultsJson, ScenarioRunResult } from './types.js';
+export declare function aggregateResults(params: {
+  runId: string;
+  timestamp: string;
+  gitCommit?: string;
+  configHash: string;
+  cliVersion: string;
+  scenarioRuns: Array<{
+    scenario_id: string;
+    scenario_name?: string;
+    agent: string;
+    eval?: EvalRules;
+    runs: ScenarioRunResult[];
+  }>;
+}): ResultsJson;
+export declare function renderSummaryMarkdown(results: ResultsJson): string;
+//# sourceMappingURL=results.d.ts.map

package/dist/results.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../src/results.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAqB,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE/F,wBAAgB,gBAAgB,CAAC,MAAM,EAAE;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,KAAK,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,SAAS,CAAC;QACjB,IAAI,EAAE,iBAAiB,EAAE,CAAC;KAC3B,CAAC,CAAC;CACJ,GAAG,WAAW,CAmGd;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CA4BlE"}

package/dist/results.js

@@ -0,0 +1,111 @@
+export function aggregateResults(params) {
+    const scenarios = params.scenarioRuns.map((entry) => {
+        const distinctSequences = {};
+        const toolUsageFrequency = {};
+        const extractedValues = {};
+        const requiredStats = {};
+        const forbiddenStats = {};
+        let lastFinalAnswer = '';
+        for (const run of entry.runs) {
+            const seqKey = JSON.stringify(run.tool_sequence);
+            distinctSequences[seqKey] = (distinctSequences[seqKey] ?? 0) + 1;
+            for (const [tool, count] of Object.entries(run.tool_usage)) {
+                toolUsageFrequency[tool] = (toolUsageFrequency[tool] ?? 0) + count;
+            }
+            if (entry.eval?.tool_constraints?.required_tools) {
+                for (const tool of entry.eval.tool_constraints.required_tools) {
+                    if (run.tool_usage[tool]) {
+                        requiredStats[tool] = (requiredStats[tool] ?? 0) + 1;
+                    }
+                    else {
+                        requiredStats[tool] ??= 0;
+                    }
+                }
+            }
+            if (entry.eval?.tool_constraints?.forbidden_tools) {
+                for (const tool of entry.eval.tool_constraints.forbidden_tools) {
+                    if (run.tool_usage[tool]) {
+                        forbiddenStats[tool] = (forbiddenStats[tool] ?? 0) + 1;
+                    }
+                    else {
+                        forbiddenStats[tool] ??= 0;
+                    }
+                }
+            }
+            for (const [name, value] of Object.entries(run.extracted)) {
+                extractedValues[name] ??= {};
+                const key = String(value);
+                extractedValues[name][key] = (extractedValues[name][key] ?? 0) + 1;
+            }
+            lastFinalAnswer = run.final_text || lastFinalAnswer;
+        }
+        const passRate = entry.runs.length === 0 ? 0 : entry.runs.filter((run) => run.pass).length / entry.runs.length;
+        const toolConstraintsStats = Object.keys(requiredStats).length > 0 || Object.keys(forbiddenStats).length > 0
+            ? { required: requiredStats, forbidden: forbiddenStats }
+            : undefined;
+        return {
+            scenario_id: entry.scenario_id,
+            scenario_name: entry.scenario_name,
+            agent: entry.agent,
+            eval: entry.eval,
+            tool_constraints_stats: toolConstraintsStats,
+            runs: entry.runs,
+            pass_rate: passRate,
+            distinct_sequences: distinctSequences,
+            tool_usage_frequency: toolUsageFrequency,
+            extracted_values: extractedValues,
+            last_final_answer: lastFinalAnswer
+        };
+    });
+    const totalRuns = scenarios.reduce((sum, scenario) => sum + scenario.runs.length, 0);
+    const totalScenarios = scenarios.length;
+    const totalPasses = scenarios.reduce((sum, scenario) => sum + scenario.runs.filter((run) => run.pass).length, 0);
+    const totalToolCalls = scenarios.reduce((sum, scenario) => sum + scenario.runs.reduce((acc, run) => acc + run.tool_call_count, 0), 0);
+    const allDurations = scenarios.flatMap((scenario) => scenario.runs.flatMap((run) => run.tool_durations_ms));
+    const avgLatency = allDurations.length === 0
+        ? null
+        : allDurations.reduce((a, b) => a + b, 0) / allDurations.length;
+    return {
+        metadata: {
+            run_id: params.runId,
+            timestamp: params.timestamp,
+            git_commit: params.gitCommit,
+            config_hash: params.configHash,
+            cli_version: params.cliVersion
+        },
+        summary: {
+            total_scenarios: totalScenarios,
+            total_runs: totalRuns,
+            pass_rate: totalRuns === 0 ? 0 : totalPasses / totalRuns,
+            avg_tool_calls_per_run: totalRuns === 0 ? 0 : totalToolCalls / totalRuns,
+            avg_tool_latency_ms: avgLatency
+        },
+        scenarios
+    };
+}
+export function renderSummaryMarkdown(results) {
+    const lines = [];
+    lines.push(`# MCP Eval Summary`);
+    lines.push('');
+    lines.push(`Run ID: ${results.metadata.run_id}`);
+    lines.push(`Timestamp: ${results.metadata.timestamp}`);
+    if (results.metadata.git_commit) {
+        lines.push(`Git commit: ${results.metadata.git_commit}`);
+    }
+    lines.push(`Config hash: ${results.metadata.config_hash}`);
+    lines.push(`CLI version: ${results.metadata.cli_version}`);
+    lines.push('');
+    lines.push(`Total scenarios: ${results.summary.total_scenarios}`);
+    lines.push(`Total runs: ${results.summary.total_runs}`);
+    lines.push(`Pass rate: ${(results.summary.pass_rate * 100).toFixed(1)}%`);
+    lines.push('');
+    lines.push('| Scenario | Agent | Runs | Pass rate | Distinct sequences | Tool calls |');
+    lines.push('|---|---|---|---|---|---|');
+    for (const scenario of results.scenarios) {
+        const passRate = (scenario.pass_rate * 100).toFixed(1);
+        const toolCalls = scenario.runs.reduce((sum, run) => sum + run.tool_call_count, 0);
+        lines.push(`| ${scenario.scenario_id} | ${scenario.agent} | ${scenario.runs.length} | ${passRate}% | ${Object.keys(scenario.distinct_sequences).length} | ${toolCalls} |`);
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=results.js.map

package/dist/results.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../src/results.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAAC,MAahC;IACC,MAAM,SAAS,GAAwB,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACvE,MAAM,iBAAiB,GAA2B,EAAE,CAAC;QACrD,MAAM,kBAAkB,GAA2B,EAAE,CAAC;QACtD,MAAM,eAAe,GAA2C,EAAE,CAAC;QACnE,MAAM,aAAa,GAA2B,EAAE,CAAC;QACjD,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,IAAI,eAAe,GAAG,EAAE,CAAC;QAEzB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACjD,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC3D,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;YACrE,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,CAAC;gBACjD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;oBAC9D,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;wBACzB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBACvD,CAAC;yBAAM,CAAC;wBACN,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC5B,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;gBAClD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC;oBAC/D,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;wBACzB,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBACzD,CAAC;yBAAM,CAAC;wBACN,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC7B,CAAC;gBACH,CAAC;YACH,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1D,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC1B,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACrE,CAAC;YACD,eAAe,GAAG,GAAG,CAAC,UAAU,IAAI,eAAe,CAAC;QACtD,CAAC;QAED,MAAM,QAAQ,GACZ,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAEhG,MAAM,oBAAoB,GACxB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC;YAC7E,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,EAAE;YACxD,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,sBAAsB,EAAE,oBAAoB;YAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,QAAQ;YACnB,kBAAkB,EAAE,iBAAiB;YACrC,oBAAoB,EAAE,kBAAkB;YACxC,gBAAgB,EAAE,eAAe;YACjC,iBAAiB,EAAE,eAAe;SACnC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC;IACxC,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,EACvE,CAAC,CACF,CAAC;IACF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,EACzF,CAAC,CACF,CAAC;IACF,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAClD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,CAAC,CACtD,CAAC;IACF,MAAM,UAAU,GACd,YAAY,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAEpE,OAAO;QACL,QAAQ,EAAE;YACR,MAAM,EAAE,MAAM,CAAC,KAAK;YACpB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,SAAS;YAC5B,WAAW,EAAE,MAAM,CAAC,UAAU;YAC9B,WAAW,EAAE,MAAM,CAAC,UAAU;SAC/B;QACD,OAAO,EAAE;YACP,eAAe,EAAE,cAAc;YAC/B,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS;YACxD,sBAAsB,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,SAAS;YACxE,mBAAmB,EAAE,UAAU;SAChC;QACD,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAoB;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;IACvD,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAClE,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACxC,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACnF,KAAK,CAAC,IAAI,CACR,KAAK,QAAQ,CAAC,WAAW,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,MAAM,QAAQ,OACnF,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,MAC3C,MAAM,SAAS,IAAI,CACpB,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,66 @@
+import type { ExecutableEvalConfig, ResultsJson } from './types.js';
+import { type AgentRunProgressEvent } from './agent.js';
+export interface RunOptions {
+  runsPerScenario: number;
+  scenarioId?: string;
+  configHash: string;
+  gitCommit?: string;
+  cliVersion: string;
+  runsDir?: string;
+  signal?: AbortSignal;
+  onProgress?: (event: RunProgressEvent) => void | Promise<void>;
+}
+export type RunProgressEvent =
+  | {
+      type: 'run_started';
+      runId: string;
+      totalScenarioRuns: number;
+      runsPerScenario: number;
+    }
+  | {
+      type: 'mcp_connect_started';
+      serverCount: number;
+    }
+  | {
+      type: 'mcp_connect_finished';
+      serverCount: number;
+    }
+  | {
+      type: 'scenario_run_started';
+      scenarioId: string;
+      agentName: string;
+      scenarioRunIndex: number;
+      totalScenarioRuns: number;
+      runIndex: number;
+      runsPerScenario: number;
+    }
+  | {
+      type: 'scenario_run_finished';
+      scenarioId: string;
+      agentName: string;
+      scenarioRunIndex: number;
+      totalScenarioRuns: number;
+      runIndex: number;
+      runsPerScenario: number;
+      pass: boolean;
+      toolCallCount: number;
+    }
+  | {
+      type: 'agent_progress';
+      scenarioRunIndex: number;
+      totalScenarioRuns: number;
+      event: AgentRunProgressEvent;
+    }
+  | {
+      type: 'run_finished';
+      runId: string;
+      totalScenarioRuns: number;
+    };
+export declare function runAll(
+  config: ExecutableEvalConfig,
+  options: RunOptions
+): Promise<{
+  runDir: string;
+  results: ResultsJson;
+}>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,oBAAoB,EAEpB,WAAW,EAGZ,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAoB,KAAK,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAI1E,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;CACzB,GACD;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GACrD;IACE,IAAI,EAAE,sBAAsB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB,GACD;IACE,IAAI,EAAE,uBAAuB,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,OAAO,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,gBAAgB,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,qBAAqB,CAAC;CAC9B,GACD;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE,wBAAsB,MAAM,CAC1B,MAAM,EAAE,oBAAoB,EAC5B,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,WAAW,CAAA;CAAE,CAAC,CA8KnD"}