@inspectr/mcplab 1.28.1 → 1.28.3

package/dist/app-server/scenario-assistant-domain.d.ts

@@ -5,180 +5,154 @@ import type { AssistantSseEvent } from './assistant-events.js';
 import { readLibraries } from './libraries-store.js';
 export { truncateJson } from './assistant-common.js';
 interface ScenarioAssistantContextInput {
-  scenario: {
-    id: string;
-    name?: string;
-    prompt: string;
-    serverNames: string[];
-    evalRules: Array<{
-      type: string;
-      value?: string | number | boolean;
-      sequence?: string[];
-      path?: string;
-      equals?: string | number | boolean;
-      tool?: string;
-      label?: string;
-      prompt?: string;
+    scenario: {
+        id: string;
+        name?: string;
+        prompt: string;
+        serverNames: string[];
+        evalRules: Array<{
+            type: string;
+            value?: string | number | boolean;
+            sequence?: string[];
+            path?: string;
+            equals?: string | number | boolean;
+            tool?: string;
+            label?: string;
+            prompt?: string;
+        }>;
+        extractRules: Array<{
+            name: string;
+            pattern: string;
+        }>;
+    };
+    availableServers?: Array<{
+        name: string;
+        url?: string;
     }>;
-    extractRules: Array<{
-      name: string;
-      pattern: string;
+    availableAgents?: Array<{
+        name: string;
+        provider: string;
+        model: string;
     }>;
-  };
-  availableServers?: Array<{
-    name: string;
-    url?: string;
-  }>;
-  availableAgents?: Array<{
-    name: string;
-    provider: string;
-    model: string;
-  }>;
 }
 interface ScenarioAssistantSuggestionBundle {
-  prompt?: {
-    replacement: string;
-    rationale?: string;
-  };
-  evalRules?: {
-    replacement: Array<{
-      type: string;
-      value?: string | number | boolean;
-      sequence?: string[];
-      path?: string;
-      equals?: string | number | boolean;
-      tool?: string;
-      label?: string;
-      prompt?: string;
-    }>;
-    rationale?: string;
-  };
-  extractRules?: {
-    replacement: Array<{
-      name: string;
-      pattern: string;
-    }>;
-    rationale?: string;
-  };
-  notes?: string[];
+    prompt?: {
+        replacement: string;
+        rationale?: string;
+    };
+    evalRules?: {
+        replacement: Array<{
+            type: string;
+            value?: string | number | boolean;
+            sequence?: string[];
+            path?: string;
+            equals?: string | number | boolean;
+            tool?: string;
+            label?: string;
+            prompt?: string;
+        }>;
+        rationale?: string;
+    };
+    extractRules?: {
+        replacement: Array<{
+            name: string;
+            pattern: string;
+        }>;
+        rationale?: string;
+    };
+    notes?: string[];
 }
 interface ScenarioAssistantEvalRuleSuggestion {
-  type: string;
-  value?: string | number | boolean;
-  sequence?: string[];
-  path?: string;
-  equals?: string | number | boolean;
-  tool?: string;
-  label?: string;
-  prompt?: string;
+    type: string;
+    value?: string | number | boolean;
+    sequence?: string[];
+    path?: string;
+    equals?: string | number | boolean;
+    tool?: string;
+    label?: string;
+    prompt?: string;
 }
 interface AssistantPendingToolCall {
-  id: string;
-  server: string;
-  tool: string;
-  publicToolName: string;
-  arguments: unknown;
-  status: 'pending' | 'approved' | 'denied' | 'error';
-  createdAt: string;
-  resultPreview?: string;
-  error?: string;
+    id: string;
+    server: string;
+    tool: string;
+    publicToolName: string;
+    arguments: unknown;
+    status: 'pending' | 'approved' | 'denied' | 'error';
+    createdAt: string;
+    resultPreview?: string;
+    error?: string;
 }
 interface AssistantChatMessage {
-  id: string;
-  role: 'user' | 'assistant' | 'tool' | 'system';
-  text: string;
-  createdAt: string;
-  suggestions?: ScenarioAssistantSuggestionBundle;
-  pendingToolCallId?: string;
-  pendingToolCallIds?: string[];
-  toolRequestServer?: string;
-  toolRequestName?: string;
-  toolRequestPublicName?: string;
+    id: string;
+    role: 'user' | 'assistant' | 'tool' | 'system';
+    text: string;
+    createdAt: string;
+    suggestions?: ScenarioAssistantSuggestionBundle;
+    pendingToolCallId?: string;
+    pendingToolCallIds?: string[];
+    toolRequestServer?: string;
+    toolRequestName?: string;
+    toolRequestPublicName?: string;
 }
 export interface ScenarioAssistantSession {
-  id: string;
-  createdAt: number;
-  lastTouchedAt: number;
-  configPath?: string;
-  selectedAssistantAgentName: string;
-  context: ScenarioAssistantContextInput;
-  agentConfig: AgentConfig;
-  mcp: McpClientManager;
-  tools: ToolDef[];
-  toolPublicMap: Map<
-    string,
-    {
-      server: string;
-      tool: string;
-    }
-  >;
-  pendingToolCalls: AssistantPendingToolCall[];
-  chatMessages: AssistantChatMessage[];
-  llmMessages: LlmMessage[];
-  warnings: string[];
-  systemPromptCache?: string;
-  events: AssistantSseEvent[];
-  clients: Set<ServerResponse>;
+    id: string;
+    createdAt: number;
+    lastTouchedAt: number;
+    configPath?: string;
+    selectedAssistantAgentName: string;
+    context: ScenarioAssistantContextInput;
+    agentConfig: AgentConfig;
+    mcp: McpClientManager;
+    tools: ToolDef[];
+    toolPublicMap: Map<string, {
+        server: string;
+        tool: string;
+    }>;
+    pendingToolCalls: AssistantPendingToolCall[];
+    chatMessages: AssistantChatMessage[];
+    llmMessages: LlmMessage[];
+    warnings: string[];
+    systemPromptCache?: string;
+    events: AssistantSseEvent[];
+    clients: Set<ServerResponse>;
 }
-export declare function cleanupAssistantSessions(
-  sessions: Map<string, ScenarioAssistantSession>,
-  now?: number
-): void;
+export declare function cleanupAssistantSessions(sessions: Map<string, ScenarioAssistantSession>, now?: number): void;
 export declare function touchAssistantSession(session: ScenarioAssistantSession): void;
 export declare function assistantSessionView(session: ScenarioAssistantSession): {
-  id: string;
-  createdAt: string;
-  updatedAt: string;
-  selectedAssistantAgentName: string;
-  model: string;
-  provider: 'openai' | 'anthropic' | 'azure_openai';
-  warnings: string[];
-  toolsLoaded: number;
-  toolServers: string[];
-  messages: AssistantChatMessage[];
-  pendingToolCalls: AssistantPendingToolCall[];
+    id: string;
+    createdAt: string;
+    updatedAt: string;
+    selectedAssistantAgentName: string;
+    model: string;
+    provider: "openai" | "anthropic" | "azure_openai";
+    warnings: string[];
+    toolsLoaded: number;
+    toolServers: string[];
+    messages: AssistantChatMessage[];
+    pendingToolCalls: AssistantPendingToolCall[];
 };
-export declare function preloadAssistantTools(
-  session: ScenarioAssistantSession,
-  serversByName: Record<string, EvalConfig['servers'][string]>,
-  selectedServerNames: string[],
-  options?: {
+export declare function preloadAssistantTools(session: ScenarioAssistantSession, serversByName: Record<string, EvalConfig['servers'][string]>, selectedServerNames: string[], options?: {
     serverAuthHeaders?: Record<string, Record<string, string>>;
-  }
-): Promise<void>;
-export declare function normalizeScenarioAssistantEvalRules(
-  replacement: ScenarioAssistantEvalRuleSuggestion[]
-): ScenarioAssistantEvalRuleSuggestion[];
-export declare function continueAssistantTurn(
-  session: ScenarioAssistantSession,
-  signal?: AbortSignal
-): Promise<{
-  session: ReturnType<typeof assistantSessionView>;
-  response: {
-    type: 'assistant_message' | 'tool_call_request';
-    text: string;
-    suggestions?: ScenarioAssistantSuggestionBundle;
-    pendingToolCall?: AssistantPendingToolCall;
-    pendingToolCalls?: AssistantPendingToolCall[];
-  };
+}): Promise<void>;
+export declare function normalizeScenarioAssistantEvalRules(replacement: ScenarioAssistantEvalRuleSuggestion[]): ScenarioAssistantEvalRuleSuggestion[];
+export declare function continueAssistantTurn(session: ScenarioAssistantSession, signal?: AbortSignal): Promise<{
+    session: ReturnType<typeof assistantSessionView>;
+    response: {
+        type: 'assistant_message' | 'tool_call_request';
+        text: string;
+        suggestions?: ScenarioAssistantSuggestionBundle;
+        pendingToolCall?: AssistantPendingToolCall;
+        pendingToolCalls?: AssistantPendingToolCall[];
+    };
 }>;
-export declare function executeAssistantToolCall(
-  session: ScenarioAssistantSession,
-  pending: AssistantPendingToolCall,
-  signal?: AbortSignal
-): Promise<unknown>;
+export declare function executeAssistantToolCall(session: ScenarioAssistantSession, pending: AssistantPendingToolCall, signal?: AbortSignal): Promise<unknown>;
 export declare function summarizeToolResultForAssistant(result: unknown): string;
-export declare function resolveAssistantAgentFromConfig(
-  config: EvalConfig,
-  selectedAssistantAgentName: string
-): AgentConfig;
-export declare function resolveAssistantAgentFromLibraries(
-  libraries: ReturnType<typeof readLibraries>,
-  selectedAssistantAgentName: string
-): AgentConfig;
+export declare function resolveAssistantAgentFromConfig(config: EvalConfig, selectedAssistantAgentName: string): AgentConfig;
+export declare function resolveAssistantAgentFromLibraries(libraries: ReturnType<typeof readLibraries>, selectedAssistantAgentName: string): AgentConfig;
 export declare function pickDefaultAssistantAgentName(params: {
-  requested?: string;
-  settingsDefault?: string;
-  agentNames: string[];
+    requested?: string;
+    settingsDefault?: string;
+    agentNames: string[];
 }): string;
-//# sourceMappingURL=scenario-assistant-domain.d.ts.map
+//# sourceMappingURL=scenario-assistant-domain.d.ts.map

package/dist/app-server/scenario-assistant-observability.d.ts

@@ -0,0 +1,25 @@
+import { RunTree } from 'langsmith';
+export interface ScenarioAssistantRequestContext {
+  sessionId: string;
+  runId?: string;
+  provider: string;
+  model: string;
+  message: string;
+}
+export interface ScenarioAssistantCounters {
+  requests: number;
+  toolCalls: number;
+  toolResults: number;
+  errors: number;
+}
+export declare function getScenarioAssistantCounters(): ScenarioAssistantCounters;
+export declare function resetScenarioAssistantCounters(): void;
+export declare function logScenarioAssistantRequest(context: ScenarioAssistantRequestContext): void;
+export declare function logScenarioAssistantTool(
+  event: 'call' | 'result' | 'error',
+  details: Record<string, unknown>
+): void;
+export declare function startScenarioAssistantLangSmithRun(
+  context: ScenarioAssistantRequestContext
+): RunTree | undefined;
+//# sourceMappingURL=scenario-assistant-observability.d.ts.map

package/dist/app-server/scenario-assistant-observability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-assistant-observability.d.ts","sourceRoot":"","sources":["../../src/app-server/scenario-assistant-observability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,OAAO,EAAE,MAAM,WAAW,CAAC;AAE5C,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAID,wBAAgB,4BAA4B,IAAI,yBAAyB,CAExE;AAED,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAED,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,+BAA+B,GAAG,IAAI,CAM1F;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,EAClC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,IAAI,CAQN;AAED,wBAAgB,kCAAkC,CAChD,OAAO,EAAE,+BAA+B,GACvC,OAAO,GAAG,SAAS,CAoBrB"}

package/dist/app-server/scenario-assistant-observability.js

@@ -0,0 +1,47 @@
+import { Client, RunTree } from 'langsmith';
+let counters = { requests: 0, toolCalls: 0, toolResults: 0, errors: 0 };
+export function getScenarioAssistantCounters() {
+    return { ...counters };
+}
+export function resetScenarioAssistantCounters() {
+    counters = { requests: 0, toolCalls: 0, toolResults: 0, errors: 0 };
+}
+export function logScenarioAssistantRequest(context) {
+    counters.requests += 1;
+    console.info('[Scenario Assistant] LLM request', {
+        ...context,
+        counters: getScenarioAssistantCounters()
+    });
+}
+export function logScenarioAssistantTool(event, details) {
+    if (event === 'call')
+        counters.toolCalls += 1;
+    if (event === 'result')
+        counters.toolResults += 1;
+    if (event === 'error')
+        counters.errors += 1;
+    console.info(`[Scenario Assistant] MCP tool ${event}`, {
+        ...details,
+        counters: getScenarioAssistantCounters()
+    });
+}
+export function startScenarioAssistantLangSmithRun(context) {
+    if (process.env.LANGSMITH_TRACING?.trim().toLowerCase() !== 'true' ||
+        !process.env.LANGSMITH_API_KEY)
+        return undefined;
+    const client = new Client({
+        apiKey: process.env.LANGSMITH_API_KEY,
+        ...(process.env.LANGSMITH_ENDPOINT ? { apiUrl: process.env.LANGSMITH_ENDPOINT } : {})
+    });
+    return new RunTree({
+        name: `MCPLab Scenario Assistant: ${context.sessionId}`,
+        run_type: 'chain',
+        inputs: { message: context.message },
+        project_name: process.env.LANGSMITH_PROJECT || 'default',
+        tags: ['mcplab', 'scenario-assistant', 'mastra'],
+        metadata: { sessionId: context.sessionId, provider: context.provider, model: context.model },
+        client,
+        tracingEnabled: true
+    });
+}
+//# sourceMappingURL=scenario-assistant-observability.js.map

package/dist/app-server/scenario-assistant-observability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-assistant-observability.js","sourceRoot":"","sources":["../../src/app-server/scenario-assistant-observability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiB5C,IAAI,QAAQ,GAA8B,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AAEnG,MAAM,UAAU,4BAA4B;IAC1C,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,8BAA8B;IAC5C,QAAQ,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAwC;IAClF,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvB,OAAO,CAAC,IAAI,CAAC,kCAAkC,EAAE;QAC/C,GAAG,OAAO;QACV,QAAQ,EAAE,4BAA4B,EAAE;KACzC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,KAAkC,EAClC,OAAgC;IAEhC,IAAI,KAAK,KAAK,MAAM;QAAE,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;IAC9C,IAAI,KAAK,KAAK,QAAQ;QAAE,QAAQ,CAAC,WAAW,IAAI,CAAC,CAAC;IAClD,IAAI,KAAK,KAAK,OAAO;QAAE,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,iCAAiC,KAAK,EAAE,EAAE;QACrD,GAAG,OAAO;QACV,QAAQ,EAAE,4BAA4B,EAAE;KACzC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,OAAwC;IAExC,IACE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,MAAM;QAC9D,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAE9B,OAAO,SAAS,CAAC;IACnB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;QACxB,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QACrC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtF,CAAC,CAAC;IACH,OAAO,IAAI,OAAO,CAAC;QACjB,IAAI,EAAE,8BAA8B,OAAO,CAAC,SAAS,EAAE;QACvD,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;QACpC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS;QACxD,IAAI,EAAE,CAAC,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC;QAChD,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QAC5F,MAAM;QACN,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;AACL,CAAC"}

package/dist/app-server/scenario-assistant-runtime.d.ts

@@ -0,0 +1,5 @@
+export type ScenarioAssistantRuntime = 'legacy' | 'mastra';
+export declare function resolveScenarioAssistantRuntime(
+  value?: string | undefined
+): ScenarioAssistantRuntime;
+//# sourceMappingURL=scenario-assistant-runtime.d.ts.map

package/dist/app-server/scenario-assistant-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-assistant-runtime.d.ts","sourceRoot":"","sources":["../../src/app-server/scenario-assistant-runtime.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,wBAAwB,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE3D,wBAAgB,+BAA+B,CAC7C,KAAK,qBAAgD,GACpD,wBAAwB,CAI1B"}

package/dist/app-server/scenario-assistant-runtime.js

@@ -0,0 +1,7 @@
+export function resolveScenarioAssistantRuntime(value = process.env.MCPLAB_SCENARIO_ASSISTANT_RUNTIME) {
+    const runtime = value?.trim().toLowerCase() || 'mastra';
+    if (runtime === 'legacy' || runtime === 'mastra')
+        return runtime;
+    throw new Error('MCPLAB_SCENARIO_ASSISTANT_RUNTIME must be either "mastra" or "legacy"');
+}
+//# sourceMappingURL=scenario-assistant-runtime.js.map

package/dist/app-server/scenario-assistant-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-assistant-runtime.js","sourceRoot":"","sources":["../../src/app-server/scenario-assistant-runtime.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,+BAA+B,CAC7C,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,iCAAiC;IAErD,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,QAAQ,CAAC;IACxD,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IACjE,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;AAC3F,CAAC"}

package/dist/app-server/types.d.ts

@@ -1,35 +1,35 @@
 import type { HealthMcpConnectionInfo } from '@inspectr/mcplab-core';
 export type { HealthMcpConnectionInfo };
 export interface AppServerOptions {
-  host: string;
-  port: number;
-  evalsDir: string;
-  runsDir: string;
-  toolAnalysisResultsDir: string;
-  librariesDir: string;
-  dev: boolean;
-  open: boolean;
+    host: string;
+    port: number;
+    evalsDir: string;
+    runsDir: string;
+    toolAnalysisResultsDir: string;
+    librariesDir: string;
+    dev: boolean;
+    open: boolean;
 }
 export interface AppSettings {
-  workspaceRoot: string;
-  evalsDir: string;
-  runsDir: string;
-  toolAnalysisResultsDir: string;
-  librariesDir: string;
-  defaultQueueWorkers: number;
-  scenarioAssistantAgentName?: string;
-  evaluationJudgeAgentName?: string;
+    workspaceRoot: string;
+    evalsDir: string;
+    runsDir: string;
+    toolAnalysisResultsDir: string;
+    librariesDir: string;
+    defaultQueueWorkers: number;
+    scenarioAssistantAgentName?: string;
+    evaluationJudgeAgentName?: string;
 }
 export interface HealthResponse {
-  ok: boolean;
-  version: string;
-  mcp: HealthMcpConnectionInfo;
+    ok: boolean;
+    version: string;
+    mcp: HealthMcpConnectionInfo;
 }
 export interface DevMcpServerRuntime {
-  host: string;
-  port: number;
-  path: string;
-  targetBaseUrl: string;
-  stop: () => void;
+    host: string;
+    port: number;
+    path: string;
+    targetBaseUrl: string;
+    stop: () => void;
 }
-//# sourceMappingURL=types.d.ts.map
+//# sourceMappingURL=types.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@inspectr/mcplab",
-  "version": "1.28.1",
+  "version": "1.28.3",
   "description": "MCPLab - Test and evaluate MCP servers with LLMs — run evals, compare agents and launch the MCPLab web app",
   "license": "MIT",
   "homepage": "https://mcplab.inspectr.dev",
@@ -44,9 +44,9 @@
     "vitest": "^4.1.8"
   },
   "dependencies": {
-    "@inspectr/mcplab-core": "1.22.1",
-    "@inspectr/mcplab-mcp-server": "1.8.1",
-    "@inspectr/mcplab-reporting": "1.2.3",
+    "@inspectr/mcplab-core": "1.22.2",
+    "@inspectr/mcplab-mcp-server": "1.8.3",
+    "@inspectr/mcplab-reporting": "1.2.4",
     "commander": "^12.1.0",
     "dotenv": "^16.6.1",
     "kleur": "^4.1.5",