@inspectr/mcplab-mcp-server 1.8.3 → 1.10.0

package/dist/runtime.d.ts

@@ -2,34 +2,28 @@ import { type IncomingMessage, type ServerResponse } from 'node:http';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 export type SessionRuntime = {
-  transport: StreamableHTTPServerTransport;
-  server: McpServer;
+    transport: StreamableHTTPServerTransport;
+    server: McpServer;
 };
 export interface McplabMcpServerOptions {
-  host: string;
-  port: number;
-  path: string;
-  logger?: Pick<Console, 'log' | 'error'>;
+    host: string;
+    port: number;
+    path: string;
+    logger?: Pick<Console, 'log' | 'error'>;
 }
 export interface McplabMcpServerRuntime {
-  host: string;
-  port: number;
-  path: string;
-  close(): Promise<void>;
+    host: string;
+    port: number;
+    path: string;
+    close(): Promise<void>;
 }
-export declare function startMcplabMcpServer(
-  options: McplabMcpServerOptions
-): Promise<McplabMcpServerRuntime>;
+export declare function startMcplabMcpServer(options: McplabMcpServerOptions): Promise<McplabMcpServerRuntime>;
 export declare function defaultMcplabMcpServerOptionsFromEnv(): McplabMcpServerOptions;
 export declare function createConfiguredServer(): McpServer;
 export declare function registerTools(server: McpServer): void;
 export declare function registerPrompts(server: McpServer): void;
-export declare function handleMcplabMcpHttpRequest(
-  req: IncomingMessage,
-  res: ServerResponse,
-  sessions: Map<string, SessionRuntime>,
-  options?: {
+export declare function handleMcplabMcpHttpRequest(req: IncomingMessage, res: ServerResponse, sessions: Map<string, SessionRuntime>, options?: {
     path?: string;
-  }
-): Promise<void>;
-//# sourceMappingURL=runtime.d.ts.map
+    logger?: Pick<Console, 'log' | 'error'>;
+}): Promise<void>;
+//# sourceMappingURL=runtime.d.ts.map

package/dist/runtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAEpF,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AA4tBnG,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,6BAA6B,CAAC;IACzC,MAAM,EAAE,SAAS,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,CAAC,CAoDjC;AAED,wBAAgB,oCAAoC,IAAI,sBAAsB,CAM7E;AAED,wBAAgB,sBAAsB,IAAI,SAAS,CAclD;AA8CD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CA4vErD;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CA0EvD;AAwsCD,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EACrC,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1B,OAAO,CAAC,IAAI,CAAC,CAEf"}
+{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAEpF,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAkvBnG,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,6BAA6B,CAAC;IACzC,MAAM,EAAE,SAAS,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,CAAC,CAoDjC;AAED,wBAAgB,oCAAoC,IAAI,sBAAsB,CAM7E;AAED,wBAAgB,sBAAsB,IAAI,SAAS,CAclD;AA8CD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CA0zErD;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CA0EvD;AAqvCD,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EACrC,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,GAAG,OAAO,CAAC,CAAA;CAAE,GACnE,OAAO,CAAC,IAAI,CAAC,CAQf"}

package/dist/runtime.decision.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=runtime.decision.test.d.ts.map

package/dist/runtime.decision.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.decision.test.d.ts","sourceRoot":"","sources":["../src/runtime.decision.test.ts"],"names":[],"mappings":""}

package/dist/runtime.decision.test.js

@@ -0,0 +1,141 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { aggregateResults } from '@inspectr/mcplab-core';
+const mocks = vi.hoisted(() => ({
+    runAll: vi.fn(),
+    loadConfig: vi.fn(),
+    loadWorkspaceSettingsOverrides: vi.fn(),
+    readLibraryAgentsAndServers: vi.fn()
+}));
+vi.mock('@inspectr/mcplab-core', async (importOriginal) => ({
+    ...(await importOriginal()),
+    ...mocks
+}));
+import { registerTools } from './runtime.js';
+describe('immediate MCP decision evaluation', () => {
+    const agent = { provider: 'openai', model: 'gpt-5-mini' };
+    const decision = {
+        provider: 'ollama',
+        model: 'clef-flash',
+        passThreshold: 0.9,
+        failThreshold: 0.1,
+        baseUrl: 'http://localhost:11434'
+    };
+    let runTool;
+    beforeEach(() => {
+        vi.clearAllMocks();
+        mocks.loadConfig.mockReturnValue({
+            config: {
+                name: 'test',
+                servers: {},
+                agents: {},
+                scenarios: [
+                    {
+                        id: 'semantic',
+                        servers: [],
+                        prompt: 'Answer',
+                        eval: { agent_assertions: [{ label: 'Check', prompt: 'Valid?' }] }
+                    }
+                ]
+            },
+            hash: 'test'
+        });
+        mocks.readLibraryAgentsAndServers.mockReturnValue({ agents: { judge: agent }, servers: {} });
+        mocks.runAll.mockResolvedValue({
+            runDir: '/tmp/mcplab-mcp-test',
+            results: aggregateResults({
+                runId: 'test',
+                timestamp: '2026-10-13T00:00:00Z',
+                configHash: 'test',
+                cliVersion: 'test',
+                scenarioRuns: []
+            })
+        });
+        registerTools({
+            registerTool: (name, _config, cb) => {
+                if (name === 'mcplab_run_eval')
+                    runTool = cb;
+                return { name };
+            }
+        });
+    });
+    it.each(['deleted', 'browser'])('runs deterministic checks despite a %s workspace judge', async (name) => {
+        mocks.loadConfig.mockReturnValue({
+            config: {
+                servers: {},
+                agents: {},
+                scenarios: [
+                    {
+                        id: 'deterministic',
+                        servers: [],
+                        prompt: 'Answer',
+                        eval: { response_assertions: [{ type: 'contains', value: 'done' }] }
+                    }
+                ]
+            }
+        });
+        mocks.loadWorkspaceSettingsOverrides.mockReturnValue({
+            evaluation_judge_agent_name: name,
+            evaluation_decision: decision
+        });
+        mocks.readLibraryAgentsAndServers.mockReturnValue({
+            agents: {
+                main: agent,
+                browser: { type: 'browser', provider: 'polo', url: 'https://polo.example' }
+            },
+            servers: {}
+        });
+        const result = await runTool({ config_path: '/tmp/eval.yaml' });
+        expect(result.isError).not.toBe(true);
+        const options = mocks.runAll.mock.calls[0][1];
+        expect(options.evaluationJudge).toBeUndefined();
+        expect(options.evaluationDecision).toBeUndefined();
+    });
+    it('uses shared workspace decision configuration and LLM judge backup', async () => {
+        mocks.loadWorkspaceSettingsOverrides.mockReturnValue({
+            evaluation_judge_agent_name: 'judge',
+            evaluation_decision: decision
+        });
+        const result = await runTool({ config_path: '/tmp/eval.yaml' });
+        expect(result.isError).not.toBe(true);
+        expect(mocks.runAll).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
+            evaluationDecision: decision,
+            evaluationJudge: { name: 'judge', agent }
+        }));
+        expect(mocks.loadWorkspaceSettingsOverrides).toHaveBeenCalledWith(mocks.readLibraryAgentsAndServers.mock.calls[0][0]);
+    });
+    it('preserves legacy judge-only workspaces', async () => {
+        mocks.loadWorkspaceSettingsOverrides.mockReturnValue({ evaluation_judge_agent_name: 'judge' });
+        await runTool({ config_path: '/tmp/eval.yaml' });
+        expect(mocks.runAll).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
+            evaluationDecision: undefined,
+            evaluationJudge: { name: 'judge', agent }
+        }));
+    });
+    it('passes malformed persisted decision settings through for shared fallback handling', async () => {
+        const malformed = { provider: 'missing', passThreshold: 2 };
+        mocks.loadWorkspaceSettingsOverrides.mockReturnValue({
+            evaluation_judge_agent_name: 'judge',
+            evaluation_decision: malformed
+        });
+        await runTool({ config_path: '/tmp/eval.yaml' });
+        expect(mocks.runAll).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
+            evaluationDecision: malformed,
+            evaluationJudge: { name: 'judge', agent }
+        }));
+    });
+    it('rejects a browser agent selected as judge before starting execution', async () => {
+        mocks.loadWorkspaceSettingsOverrides.mockReturnValue({
+            evaluation_judge_agent_name: 'judge',
+            evaluation_decision: decision
+        });
+        mocks.readLibraryAgentsAndServers.mockReturnValue({
+            agents: { judge: { type: 'browser', provider: 'polo', url: 'https://polo.example' } },
+            servers: {}
+        });
+        const result = await runTool({ config_path: '/tmp/eval.yaml' });
+        expect(result.isError).toBe(true);
+        expect(result.content[0].text).toContain('LLM');
+        expect(mocks.runAll).not.toHaveBeenCalled();
+    });
+});
+//# sourceMappingURL=runtime.decision.test.js.map

package/dist/runtime.decision.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.decision.test.js","sourceRoot":"","sources":["../src/runtime.decision.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9B,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE;IACf,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE;IACnB,8BAA8B,EAAE,EAAE,CAAC,EAAE,EAAE;IACvC,2BAA2B,EAAE,EAAE,CAAC,EAAE,EAAE;CACrC,CAAC,CAAC,CAAC;AACJ,EAAE,CAAC,IAAI,CAAC,uBAAuB,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC,CAAC;IAC1D,GAAG,CAAC,MAAM,cAAc,EAA0C,CAAC;IACnE,GAAG,KAAK;CACT,CAAC,CAAC,CAAC;AACJ,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;IACjD,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,QAAiB,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;IACnE,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE,QAAiB;QAC3B,KAAK,EAAE,YAAY;QACnB,aAAa,EAAE,GAAG;QAClB,aAAa,EAAE,GAAG;QAClB,OAAO,EAAE,wBAAwB;KAClC,CAAC;IACF,IAAI,OAAwD,CAAC;IAE7D,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC;YAC/B,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE;oBACT;wBACE,EAAE,EAAE,UAAU;wBACd,OAAO,EAAE,EAAE;wBACX,MAAM,EAAE,QAAQ;wBAChB,IAAI,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE;qBACnE;iBACF;aACF;YACD,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QACH,KAAK,CAAC,2BAA2B,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7F,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC;YAC7B,MAAM,EAAE,sBAAsB;YAC9B,OAAO,EAAE,gBAAgB,CAAC;gBACxB,KAAK,EAAE,MAAM;gBACb,SAAS,EAAE,sBAAsB;gBACjC,UAAU,EAAE,MAAM;gBAClB,UAAU,EAAE,MAAM;gBAClB,YAAY,EAAE,EAAE;aACjB,CAAC;SACH,CAAC,CAAC;QACH,aAAa,CAAC;YACZ,YAAY,EAAE,CAAC,IAAY,EAAE,OAAgB,EAAE,EAAkB,EAAE,EAAE;gBACnE,IAAI,IAAI,KAAK,iBAAiB;oBAAE,OAAO,GAAG,EAAE,CAAC;gBAC7C,OAAO,EAAE,IAAI,EAAE,CAAC;YAClB,CAAC;SACK,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAC7B,wDAAwD,EACxD,KAAK,EAAE,IAAI,EAAE,EAAE;QACb,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC;YAC/B,MAAM,EAAE;gBACN,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE;oBACT;wBACE,EAAE,EAAE,eAAe;wBACnB,OAAO,EAAE,EAAE;wBACX,MAAM,EAAE,QAAQ;wBAChB,IAAI,EAAE,EAAE,mBAAmB,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;qBACrE;iBACF;aACF;SACF,CAAC,CAAC;QACH,KAAK,CAAC,8BAA8B,CAAC,eAAe,CAAC;YACnD,2BAA2B,EAAE,IAAI;YACjC,mBAAmB,EAAE,QAAQ;SAC9B,CAAC,CAAC;QACH,KAAK,CAAC,2BAA2B,CAAC,eAAe,CAAC;YAChD,MAAM,EAAE;gBACN,IAAI,EAAE,KAAK;gBACX,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,sBAAsB,EAAE;aAC5E;YACD,OAAO,EAAE,EAAE;SACZ,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,aAAa,EAAE,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,aAAa,EAAE,CAAC;IACrD,CAAC,CACF,CAAC;IAEF,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACjF,KAAK,CAAC,8BAA8B,CAAC,eAAe,CAAC;YACnD,2BAA2B,EAAE,OAAO;YACpC,mBAAmB,EAAE,QAAQ;SAC9B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,oBAAoB,CACvC,MAAM,CAAC,QAAQ,EAAE,EACjB,MAAM,CAAC,gBAAgB,CAAC;YACtB,kBAAkB,EAAE,QAAQ;YAC5B,eAAe,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C,CAAC,CACH,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,oBAAoB,CAC/D,KAAK,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACnD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,KAAK,CAAC,8BAA8B,CAAC,eAAe,CAAC,EAAE,2BAA2B,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/F,MAAM,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,oBAAoB,CACvC,MAAM,CAAC,QAAQ,EAAE,EACjB,MAAM,CAAC,gBAAgB,CAAC;YACtB,kBAAkB,EAAE,SAAS;YAC7B,eAAe,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,KAAK,IAAI,EAAE;QACjG,MAAM,SAAS,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,8BAA8B,CAAC,eAAe,CAAC;YACnD,2BAA2B,EAAE,OAAO;YACpC,mBAAmB,EAAE,SAAS;SAC/B,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,oBAAoB,CACvC,MAAM,CAAC,QAAQ,EAAE,EACjB,MAAM,CAAC,gBAAgB,CAAC;YACtB,kBAAkB,EAAE,SAAS;YAC7B,eAAe,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,KAAK,CAAC,8BAA8B,CAAC,eAAe,CAAC;YACnD,2BAA2B,EAAE,OAAO;YACpC,mBAAmB,EAAE,QAAQ;SAC9B,CAAC,CAAC;QACH,KAAK,CAAC,2BAA2B,CAAC,eAAe,CAAC;YAChD,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,sBAAsB,EAAE,EAAE;YACrF,OAAO,EAAE,EAAE;SACZ,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/runtime.js

@@ -5,7 +5,7 @@ import { basename, dirname, extname, resolve, join, sep, relative } from 'node:p
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
-import { loadConfig, hashConfig, runAll, selectScenarios, loadOrBuildSearchIndex, indexNeedsRefresh, getResultsIndexPaths, resolveRunArtifactPath, searchDocs, getContext, applyRuntimeServerOverrides, readLibraryAgentsAndServers, resolveScenarioLibraryDir, createEvaluationConfigFile, updateEvaluationConfigFile, readEvaluationConfigFile, createTestCaseFile, readTestCaseFile, updateTestCaseFile, buildScenarioEntry } from '@inspectr/mcplab-core';
+import { loadConfig, hashConfig, runAll, loadWorkspaceSettingsOverrides, toWorkspaceEvaluationSettings, resolveWorkspaceEvaluation, selectScenarios, loadOrBuildSearchIndex, indexNeedsRefresh, getResultsIndexPaths, resolveRunArtifactPath, searchDocs, getContext, applyRuntimeServerOverrides, readLibraryAgentsAndServers, readBrowserProviderProfiles, serializeBrowserProviderProfile, resolveScenarioLibraryDir, createEvaluationConfigFile, updateEvaluationConfigFile, readEvaluationConfigFile, createTestCaseFile, readTestCaseFile, updateTestCaseFile, buildScenarioEntry, DEFAULT_BROWSER_PROVIDER_PROFILES, mergeBrowserProviderProfiles } from '@inspectr/mcplab-core';
 import { renderReport } from '@inspectr/mcplab-reporting';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
 import { z } from 'zod';
@@ -25,7 +25,7 @@ const GenericObjectSchema = z.object({}).passthrough();
 const ResultsSummarySchema = z.object({
     total_scenarios: z.number().int().nonnegative(),
     total_runs: z.number().int().nonnegative(),
-    pass_rate: z.number(),
+    pass_rate: z.number().nullable(),
     avg_tool_calls_per_run: z.number(),
     avg_tool_latency_ms: z.number().nullable()
 });
@@ -44,7 +44,8 @@ const MetricSummarySchema = z.object({
     total_runs: z.number().int().nonnegative(),
     passed_runs: z.number().int().nonnegative(),
     failed_runs: z.number().int().nonnegative(),
-    pass_rate: z.number(),
+    stopped_runs: z.number().int().nonnegative().optional(),
+    pass_rate: z.number().nullable(),
     avg_tool_calls_per_run: z.number(),
     avg_tool_latency_ms: z.number().nullable()
 });
@@ -66,7 +67,7 @@ const CompareRowSchema = z.object({
     key: z.string(),
     scenario_id: z.string(),
     agent: z.string(),
-    classification: z.enum(['regressed', 'improved', 'unchanged', 'new', 'missing']),
+    classification: z.enum(['regressed', 'improved', 'unchanged', 'new', 'missing', 'not_evaluated']),
     left: MetricSummarySchema.nullable(),
     right: MetricSummarySchema.nullable(),
     deltas: z.object({
@@ -90,12 +91,19 @@ const LibraryScenarioEntrySchema = z.object({
     yaml: z.string().optional()
 });
 const AgentEntrySchema = z.object({
-    provider: z.enum(['openai', 'anthropic', 'azure_openai']),
+    provider: z.enum(['openai', 'anthropic', 'azure_openai', 'azure_foundry']),
     model: z.string(),
     temperature: z.number().optional(),
     max_tokens: z.number().int().positive().optional(),
     system: z.string().optional()
 });
+const BrowserAgentEntrySchema = z.object({
+    type: z.literal('browser'),
+    name: z.string().optional(),
+    provider: z.string(),
+    url: z.string(),
+    new_conversation_between_scenarios: z.boolean().optional()
+});
 const LibraryServerEntryContentSchema = z
     .object({
     transport: z.string().optional(),
@@ -112,12 +120,17 @@ const LibraryServerEntrySchema = z.object({
 });
 const LibraryAgentEntrySchema = z.object({
     id: z.string(),
-    entry: AgentEntrySchema.optional()
+    entry: z.union([AgentEntrySchema, BrowserAgentEntrySchema]).optional()
+});
+const LibraryBrowserProviderEntrySchema = z.object({
+    id: z.string(),
+    entry: GenericObjectSchema.optional()
 });
 const LibraryEntrySchema = z.object({
     bundleRoot: z.string(),
     servers: z.array(LibraryServerEntrySchema),
     agents: z.array(LibraryAgentEntrySchema),
+    browser_providers: z.array(LibraryBrowserProviderEntrySchema),
     test_cases: z.array(LibraryScenarioEntrySchema),
     scenarios: z.array(LibraryScenarioEntrySchema)
 });
@@ -615,7 +628,7 @@ export async function startMcplabMcpServer(options) {
     const sessions = new Map();
     const httpServer = createServer(async (req, res) => {
         try {
-            await handleHttpRequest(req, res, sessions, options.path);
+            await handleHttpRequest(req, res, sessions, options.path, logger);
         }
         catch (error) {
             logger.error('[mcplab-mcp] request error:', error);
@@ -1081,7 +1094,7 @@ export function registerTools(server) {
         outputSchema: LibraryEntrySchema,
         inputSchema: {
             kind: z
-                .enum(['all', 'servers', 'agents', 'test_cases', 'scenarios'])
+                .enum(['all', 'servers', 'agents', 'browser_providers', 'test_cases', 'scenarios'])
                 .optional()
                 .describe('Which library category to list. Defaults to all.'),
             includeContent: z
@@ -1091,15 +1104,16 @@ export function registerTools(server) {
         }
     }, async ({ kind, includeContent }) => {
         return withToolHandling(async () => {
-            const root = resolveBundleRoot();
-            const data = readLibrary(root, Boolean(includeContent));
             const selectedKind = kind ?? 'all';
+            const root = resolveBundleRoot();
+            const data = readLibrary(root, Boolean(includeContent), selectedKind);
             const structured = selectedKind === 'all'
                 ? data
                 : {
                     bundleRoot: data.bundleRoot,
                     servers: selectedKind === 'servers' ? data.servers : [],
                     agents: selectedKind === 'agents' ? data.agents : [],
+                    browser_providers: selectedKind === 'browser_providers' ? data.browser_providers : [],
                     test_cases: selectedKind === 'test_cases' ? data.test_cases : [],
                     scenarios: selectedKind === 'scenarios' ? data.scenarios : []
                 };
@@ -1107,10 +1121,10 @@ export function registerTools(server) {
         });
     });
     registerTool('mcplab_get_library_item', {
-        description: 'Get a specific reusable server, agent, or scenario definition from a MCPLab library bundle and return both structured data and YAML.',
+        description: 'Get a specific reusable server, agent, browser provider, test case, or scenario definition from a MCPLab library bundle and return both structured data and YAML.',
         outputSchema: {
             bundleRoot: z.string(),
-            kind: z.enum(['servers', 'agents', 'test_cases', 'scenarios']),
+            kind: z.enum(['servers', 'agents', 'browser_providers', 'test_cases', 'scenarios']),
             id: z.string(),
             file: z.string().optional(),
             yaml: z.string(),
@@ -1118,7 +1132,7 @@ export function registerTools(server) {
         },
         inputSchema: {
             kind: z
-                .enum(['servers', 'agents', 'test_cases', 'scenarios'])
+                .enum(['servers', 'agents', 'browser_providers', 'test_cases', 'scenarios'])
                 .describe('Library category.'),
             id: z.string().describe('Entry id (for scenarios this is scenario.id, not filename).')
         }
@@ -1158,9 +1172,11 @@ export function registerTools(server) {
         inputSchema: {
             id: z.string().describe('Agent id key (kebab-case recommended).'),
             provider: z
-                .enum(['openai', 'anthropic', 'azure_openai'])
+                .enum(['openai', 'anthropic', 'azure_openai', 'azure_foundry'])
                 .describe('LLM provider supported by MCPLab.'),
-            model: z.string().describe('Model id or deployment name (for Azure OpenAI).'),
+            model: z
+                .string()
+                .describe('Model id or deployment name (for Azure OpenAI and Azure Foundry).'),
             temperature: z.number().optional().describe('Sampling temperature.'),
             max_tokens: z.number().int().positive().optional().describe('Maximum output tokens.'),
             system: z.string().optional().describe('Optional system prompt.')
@@ -1175,6 +1191,46 @@ export function registerTools(server) {
             });
         });
     });
+    registerTool('mcplab_generate_browser_agent_entry', {
+        description: 'Generate an agents.yaml entry for a Rover browser agent. Browser agents reference a validated browser provider profile and are executed by Rover, not by an LLM API.',
+        outputSchema: {
+            id: z.string(),
+            entry: z.object({
+                type: z.literal('browser'),
+                name: z.string().optional(),
+                provider: z.string(),
+                url: z.string(),
+                new_conversation_between_scenarios: z.boolean().optional()
+            }),
+            yaml: z.string()
+        },
+        inputSchema: {
+            id: z.string().describe('Browser agent id key (kebab-case recommended).'),
+            name: z.string().optional().describe('Human-readable browser agent name.'),
+            provider: z
+                .string()
+                .min(1)
+                .describe('Provider id from browser-providers/<provider-id>.yaml.'),
+            url: z.string().url().describe('Browser origin where Rover should execute the agent.'),
+            new_conversation_between_scenarios: z
+                .boolean()
+                .optional()
+                .describe('Whether Rover should start a new conversation between scenarios.')
+        }
+    }, async ({ id, name, provider, url, new_conversation_between_scenarios }) => withToolHandling(async () => {
+        const entry = removeUndefined({
+            type: 'browser',
+            name,
+            provider,
+            url,
+            new_conversation_between_scenarios
+        });
+        return ok(`Generated browser agent entry '${id}'`, {
+            id,
+            entry,
+            yaml: stringifyYaml({ [id]: entry }).trimEnd()
+        });
+    }));
     registerTool('mcplab_create_test_case', {
         description: 'Create one reviewed MCPLab Test Case in mcplab/test-cases/. Put evaluation checks in the eval object for full support of response, tool-input, and agent assertions. Simple shorthand fields are supported for basic cases; do not provide a shorthand field together with its equivalent eval field. This is a scoped YAML write and refuses duplicate IDs.',
         outputSchema: { id: z.string(), path: z.string(), test_case: ScenarioEntrySchema },
@@ -1529,7 +1585,7 @@ export function registerTools(server) {
             const loaded = loadConfig(resolve(config_path), {
                 bundleRoot
             });
-            const mergedConfig = mergeLibraryServersAndAgents(loaded.config, bundleRoot);
+            const mergedConfig = mergeLibraryServersAndAgents(loaded.config, readLibraryAgentsAndServers(bundleRoot));
             const selected = selectScenarios(mergedConfig, scenario_id);
             const summary = summarizeConfig(selected);
             return ok(`Validated config ${config_path}`, {
@@ -1556,7 +1612,7 @@ export function registerTools(server) {
             scenarios: z.array(z.object({
                 scenario_id: z.string(),
                 agent: z.string(),
-                pass_rate: z.number(),
+                pass_rate: z.number().nullable(),
                 tool_usage_frequency: z.record(z.number())
             })),
             effective_server_overrides: z.record(z.array(z.string())),
@@ -1586,7 +1642,8 @@ export function registerTools(server) {
             const loaded = loadConfig(resolve(config_path), {
                 bundleRoot
             });
-            const mergedConfig = mergeLibraryServersAndAgents(loaded.config, bundleRoot);
+            const libraries = readLibraryAgentsAndServers(bundleRoot);
+            const mergedConfig = mergeLibraryServersAndAgents(loaded.config, libraries);
             const selected = selectScenarios(mergedConfig, scenario_id);
             const runtimeOverridden = applyRuntimeServerOverrides(selected, {
                 serverOverrideAll: server_override_all,
@@ -1594,13 +1651,19 @@ export function registerTools(server) {
             });
             const effectiveConfigHash = hashConfig(runtimeOverridden);
             const executable = expandConfigForAgents(runtimeOverridden, runtimeOverridden.run_defaults?.selected_agents);
+            const workspaceSettings = loadWorkspaceSettingsOverrides(bundleRoot);
             const { runDir, results } = await runAll(executable, {
                 runsPerScenario: runs_per_scenario ?? 1,
                 scenarioId: scenario_id,
                 configHash: effectiveConfigHash,
                 cliVersion: `mcplab-mcp-server/${SERVER_VERSION}`,
                 runsDir: resolveRunsDir(),
-                cwd: process.cwd()
+                cwd: process.cwd(),
+                ...resolveWorkspaceEvaluation({
+                    agents: libraries.agents,
+                    scenarios: executable.scenarios,
+                    settings: toWorkspaceEvaluationSettings(workspaceSettings)
+                })
             });
             const reportHtml = renderReport(results);
             const allRuns = results.scenarios.flatMap((scenario) => scenario.runs);
@@ -1663,7 +1726,7 @@ export function registerTools(server) {
             throw new Error('since must be earlier than or equal to until');
         const matching = listRunsWithFallback(resolveRunsDir(), undefined, true).filter((entry) => {
             const timestamp = String(entry.metadata && typeof entry.metadata === 'object'
-                ? entry.metadata.timestamp ?? ''
+                ? (entry.metadata.timestamp ?? '')
                 : '');
             const time = new Date(timestamp).getTime();
             if (sinceMs === undefined && untilMs === undefined)
@@ -1857,7 +1920,7 @@ export function registerTools(server) {
                 left: MetricSummarySchema,
                 right: MetricSummarySchema,
                 deltas: z.object({
-                    pass_rate: z.number(),
+                    pass_rate: z.number().nullable(),
                     failed_runs: z.number(),
                     avg_tool_calls_per_run: z.number(),
                     avg_tool_latency_ms: z.number().nullable()
@@ -1867,7 +1930,8 @@ export function registerTools(server) {
                     improved: z.number().int().nonnegative(),
                     unchanged: z.number().int().nonnegative(),
                     new: z.number().int().nonnegative(),
-                    missing: z.number().int().nonnegative()
+                    missing: z.number().int().nonnegative(),
+                    not_evaluated: z.number().int().nonnegative()
                 })
             }),
             regressions: z.array(CompareRowSchema),
@@ -2389,7 +2453,7 @@ export function registerTools(server) {
                 .array(z.object({
                 scenario_id: z.string(),
                 agent: z.string(),
-                pass_rate: z.number()
+                pass_rate: z.number().nullable()
             }))
                 .optional()
         },
@@ -2797,23 +2861,25 @@ function resolveServerOwnedRoots() {
 function resolveBundleRoot() {
     return SERVER_OWNED_ROOTS.bundleRoot;
 }
-function mergeLibraryServersAndAgents(config, bundleRoot) {
-    const libraries = readLibraryAgentsAndServers(bundleRoot);
+function mergeLibraryServersAndAgents(config, libraries) {
     return {
         ...config,
         agents: { ...libraries.agents, ...config.agents },
         servers: { ...libraries.servers, ...config.servers }
     };
 }
-function readLibrary(bundleRoot, includeContent) {
+function readLibrary(bundleRoot, includeContent, kind = 'all') {
     const serversPath = join(bundleRoot, 'servers.yaml');
     const agentsPath = join(bundleRoot, 'agents.yaml');
     const scenariosDir = resolveScenarioLibraryDir(bundleRoot).path;
     const servers = existsSync(serversPath)
-        ? parseYaml(readFileSync(serversPath, 'utf8')) ?? {}
+        ? (parseYaml(readFileSync(serversPath, 'utf8')) ?? {})
         : {};
     const agents = existsSync(agentsPath)
-        ? parseYaml(readFileSync(agentsPath, 'utf8')) ?? {}
+        ? (parseYaml(readFileSync(agentsPath, 'utf8')) ?? {})
+        : {};
+    const browserProviders = kind === 'all' || kind === 'browser_providers'
+        ? mergeBrowserProviderProfiles(readBrowserProviderProfiles(bundleRoot))
         : {};
     const scenarioEntries = [];
     if (existsSync(scenariosDir)) {
@@ -2845,29 +2911,57 @@ function readLibrary(bundleRoot, includeContent) {
             id,
             ...(includeContent ? { entry: agents[id] } : {})
         })),
+        browser_providers: Object.keys(browserProviders)
+            .sort()
+            .map((id) => ({
+            id,
+            ...(includeContent
+                ? { entry: browserProviders[id] }
+                : {})
+        })),
         test_cases: scenarioEntries,
         scenarios: scenarioEntries
     };
     return out;
 }
 function getLibraryItem(bundleRoot, kind, id) {
-    if (kind === 'servers' || kind === 'agents') {
-        const file = join(bundleRoot, `${kind}.yaml`);
+    if (kind === 'servers' || kind === 'agents' || kind === 'browser_providers') {
+        const directory = kind === 'browser_providers' ? join(bundleRoot, 'browser-providers') : null;
+        const builtIn = kind === 'browser_providers' && Object.hasOwn(DEFAULT_BROWSER_PROVIDER_PROFILES, id)
+            ? DEFAULT_BROWSER_PROVIDER_PROFILES[id]
+            : undefined;
+        const file = directory
+            ? findBrowserProviderFile(directory, id)
+            : join(bundleRoot, `${kind}.yaml`);
+        if (builtIn && !existsSync(file)) {
+            const content = structuredClone(builtIn);
+            return {
+                bundleRoot,
+                kind,
+                id,
+                yaml: serializeBrowserProviderProfile(content).trimEnd(),
+                content: content
+            };
+        }
         if (!existsSync(file)) {
             throw new Error(`Library file not found: ${file}`);
         }
         const raw = readFileSync(file, 'utf8');
         const parsed = parseYaml(raw) ?? {};
-        if (!(id in parsed)) {
+        if (kind !== 'browser_providers' && !(id in parsed)) {
             throw new Error(`'${id}' not found in ${file}`);
         }
-        const entry = parsed[id];
+        const normalizedEntry = kind === 'browser_providers' ? readBrowserProviderProfiles(bundleRoot)[id] : parsed[id];
+        if (!normalizedEntry)
+            throw new Error(`'${id}' not found in ${file}`);
         return {
             bundleRoot,
             kind,
             id,
-            yaml: stringifyYaml({ [id]: entry }).trimEnd(),
-            content: entry
+            yaml: kind === 'browser_providers'
+                ? raw.trimEnd()
+                : stringifyYaml({ [id]: normalizedEntry }).trimEnd(),
+            content: normalizedEntry
         };
     }
     const dir = resolveScenarioLibraryDir(bundleRoot).path;
@@ -2892,6 +2986,14 @@ function getLibraryItem(bundleRoot, kind, id) {
     }
     throw new Error(`Scenario '${id}' not found in ${dir}`);
 }
+function findBrowserProviderFile(directory, id) {
+    if (!existsSync(directory))
+        return join(directory, `${id}.yaml`);
+    const file = readdirSync(directory)
+        .filter((name) => name.endsWith('.yaml') || name.endsWith('.yml'))
+        .find((name) => basename(name, extname(name)) === id);
+    return join(directory, file ?? `${id}.yaml`);
+}
 function listEvaluationConfigs(params) {
     const evalsDir = join(resolveBundleRoot(), 'evals');
     const files = [];
@@ -3762,9 +3864,9 @@ function truncate(text, maxChars) {
     return `${text.slice(0, maxChars)}\n...[truncated ${text.length - maxChars} chars]`;
 }
 export async function handleMcplabMcpHttpRequest(req, res, sessions, options) {
-    await handleHttpRequest(req, res, sessions, options?.path ?? DEFAULT_MCP_PATH);
+    await handleHttpRequest(req, res, sessions, options?.path ?? DEFAULT_MCP_PATH, options?.logger ?? console);
 }
-async function handleHttpRequest(req, res, sessions, mcpPath) {
+async function handleHttpRequest(req, res, sessions, mcpPath, logger = console) {
     const method = req.method ?? 'GET';
     const pathname = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`).pathname;
     if (pathname === '/' && method === 'GET') {
@@ -3796,14 +3898,25 @@ async function handleHttpRequest(req, res, sessions, mcpPath) {
             return;
         }
         if (isInitializeRequest(body)) {
+            const clientInfo = body.params && typeof body.params === 'object' && 'clientInfo' in body.params
+                ? body.params.clientInfo
+                : undefined;
+            const clientName = clientInfo && typeof clientInfo === 'object' && 'name' in clientInfo
+                ? String(clientInfo.name)
+                : 'unknown';
+            const clientVersion = clientInfo && typeof clientInfo === 'object' && 'version' in clientInfo
+                ? String(clientInfo.version)
+                : undefined;
             let runtime;
             const transport = new StreamableHTTPServerTransport({
                 sessionIdGenerator: () => randomUUID(),
                 onsessioninitialized: (sid) => {
                     sessions.set(sid, runtime);
+                    logger.log(`[mcplab-mcp] client connected: ${clientName}${clientVersion ? `/${clientVersion}` : ''} (session ${sid})`);
                 },
                 onsessionclosed: (sid) => {
                     sessions.delete(sid);
+                    logger.log(`[mcplab-mcp] client disconnected: session ${sid}`);
                 }
             });
             const mcpServer = createConfiguredServer();