@inspectr/mcplab-core 1.22.1 → 1.23.0

package/dist/types.d.ts

@@ -1,442 +1,457 @@
 export type TransportType = 'http';
 export interface ServerAuthBearer {
-  type: 'bearer';
-  env?: string;
-  token?: string;
+    type: 'bearer';
+    env?: string;
+    token?: string;
 }
 export interface ServerAuthApiKey {
-  type: 'api_key';
-  header_name?: string;
-  value: string;
+    type: 'api_key';
+    header_name?: string;
+    value: string;
 }
 export interface ServerAuthOauthClientCredentials {
-  type: 'oauth_client_credentials';
-  token_url: string;
-  client_id_env: string;
-  client_secret_env: string;
-  scope?: string;
-  audience?: string;
-  token_params?: Record<string, string>;
+    type: 'oauth_client_credentials';
+    token_url: string;
+    client_id_env: string;
+    client_secret_env: string;
+    scope?: string;
+    audience?: string;
+    token_params?: Record<string, string>;
 }
 export interface ServerAuthOauthAuthorizationCode {
-  type: 'oauth_authorization_code';
-  mode?: 'pre_registered' | 'dcr';
-  client_id?: string;
-  client_secret?: string;
-  redirect_url?: string;
-  scope?: string;
-  authorization_url?: string;
-  token_url?: string;
-}
-export type ServerAuth =
-  | ServerAuthBearer
-  | ServerAuthApiKey
-  | ServerAuthOauthClientCredentials
-  | ServerAuthOauthAuthorizationCode;
+    type: 'oauth_authorization_code';
+    mode?: 'pre_registered' | 'dcr';
+    client_id?: string;
+    client_secret?: string;
+    redirect_url?: string;
+    scope?: string;
+    authorization_url?: string;
+    token_url?: string;
+}
+export type ServerAuth = ServerAuthBearer | ServerAuthApiKey | ServerAuthOauthClientCredentials | ServerAuthOauthAuthorizationCode;
 export interface ServerConfig {
-  name?: string;
-  transport: TransportType;
-  url: string;
-  headers?: Record<string, string>;
-  auth?: ServerAuth;
+    name?: string;
+    transport: TransportType;
+    url: string;
+    headers?: Record<string, string>;
+    auth?: ServerAuth;
 }
 export interface ServerInlineEntry extends ServerConfig {
-  id: string;
-  name?: string;
+    id: string;
+    name?: string;
 }
 export interface ServerRefEntry {
-  ref: string;
+    ref: string;
 }
 export type ServerListEntry = ServerInlineEntry | ServerRefEntry;
-export interface AgentConfig {
-  name?: string;
-  provider: 'openai' | 'anthropic' | 'azure_openai';
-  model: string;
-  temperature?: number;
-  max_tokens?: number;
-  max_turns?: number;
-  system?: string;
-}
-export interface AgentInlineEntry extends AgentConfig {
-  id: string;
-  name?: string;
-}
+export interface LlmAgentConfig {
+    type?: 'llm';
+    name?: string;
+    provider: 'openai' | 'anthropic' | 'azure_openai';
+    model: string;
+    temperature?: number;
+    max_tokens?: number;
+    max_turns?: number;
+    system?: string;
+}
+export interface BrowserAgentConfig {
+    type: 'browser';
+    name?: string;
+    provider: string;
+    url: string;
+    newConversationBetweenScenarios?: boolean;
+}
+export type RoverAgentProvider = string;
+export interface RoverAgentRef {
+    name: string;
+    provider: RoverAgentProvider;
+    url: string;
+    providerRevision?: string;
+}
+export type AgentConfig = LlmAgentConfig | BrowserAgentConfig;
+export type AgentInlineEntry = (LlmAgentConfig | BrowserAgentConfig) & {
+    id: string;
+    name?: string;
+};
 export interface AgentRefEntry {
-  ref: string;
+    ref: string;
 }
 export type AgentListEntry = AgentInlineEntry | AgentRefEntry;
+export declare function isLlmAgent(agent: AgentConfig): agent is LlmAgentConfig;
 export interface ToolConstraints {
-  required_tools?: string[];
-  forbidden_tools?: string[];
+    required_tools?: string[];
+    forbidden_tools?: string[];
 }
 export interface ResponseAssertionRegex {
-  type: 'regex';
-  pattern: string;
+    type: 'regex';
+    pattern: string;
 }
 export interface ResponseAssertionContains {
-  type: 'contains';
-  value: string;
+    type: 'contains';
+    value: string;
 }
 export interface ResponseAssertionNotContains {
-  type: 'not_contains';
-  value: string;
+    type: 'not_contains';
+    value: string;
 }
 export interface ResponseAssertionStartsWith {
-  type: 'starts_with';
-  value: string;
+    type: 'starts_with';
+    value: string;
 }
 export interface ResponseAssertionEndsWith {
-  type: 'ends_with';
-  value: string;
+    type: 'ends_with';
+    value: string;
 }
 export interface ResponseAssertionEquals {
-  type: 'equals';
-  value: string;
+    type: 'equals';
+    value: string;
 }
 export interface ResponseAssertionJsonPath {
-  type: 'jsonpath';
-  path: string;
-  equals?: string | number | boolean;
+    type: 'jsonpath';
+    path: string;
+    equals?: string | number | boolean;
 }
 export interface ResponseAssertionJsonPathExists {
-  type: 'jsonpath_exists';
-  path: string;
+    type: 'jsonpath_exists';
+    path: string;
 }
 export interface ResponseAssertionJsonPathNotExists {
-  type: 'jsonpath_not_exists';
-  path: string;
+    type: 'jsonpath_not_exists';
+    path: string;
 }
 export interface AgentAssertion {
-  label: string;
-  prompt: string;
+    label: string;
+    prompt: string;
 }
 export interface ToolInputAssertionBase {
-  tool: string;
+    tool: string;
 }
 export interface ToolInputAssertionContains extends ToolInputAssertionBase {
-  type: 'contains';
-  value: string;
+    type: 'contains';
+    value: string;
 }
 export interface ToolInputAssertionRegex extends ToolInputAssertionBase {
-  type: 'regex';
-  pattern: string;
+    type: 'regex';
+    pattern: string;
 }
 export interface ToolInputAssertionJsonPath extends ToolInputAssertionBase {
-  type: 'jsonpath';
-  path: string;
-  equals?: string | number | boolean;
-}
-export type ToolInputAssertion =
-  | ToolInputAssertionContains
-  | ToolInputAssertionRegex
-  | ToolInputAssertionJsonPath;
+    type: 'jsonpath';
+    path: string;
+    equals?: string | number | boolean;
+}
+export type ToolInputAssertion = ToolInputAssertionContains | ToolInputAssertionRegex | ToolInputAssertionJsonPath;
 export interface AgentContext {
-  include_prompt?: boolean;
-  include_tool_sequence?: boolean;
-  include_tool_inputs?: boolean;
+    include_prompt?: boolean;
+    include_tool_sequence?: boolean;
+    include_tool_inputs?: boolean;
 }
 export interface AgentJudgeContext {
-  scenario_prompt?: string;
-  tool_sequence?: string[];
-  tool_inputs?: Array<{
-    tool: string;
-    arguments: unknown;
-  }>;
-}
-export type ResponseAssertion =
-  | ResponseAssertionRegex
-  | ResponseAssertionContains
-  | ResponseAssertionNotContains
-  | ResponseAssertionStartsWith
-  | ResponseAssertionEndsWith
-  | ResponseAssertionEquals
-  | ResponseAssertionJsonPath
-  | ResponseAssertionJsonPathExists
-  | ResponseAssertionJsonPathNotExists;
+    scenario_prompt?: string;
+    tool_sequence?: string[];
+    tool_inputs?: Array<{
+        tool: string;
+        arguments: unknown;
+    }>;
+}
+export type ResponseAssertion = ResponseAssertionRegex | ResponseAssertionContains | ResponseAssertionNotContains | ResponseAssertionStartsWith | ResponseAssertionEndsWith | ResponseAssertionEquals | ResponseAssertionJsonPath | ResponseAssertionJsonPathExists | ResponseAssertionJsonPathNotExists;
 export interface EvalRules {
-  tool_constraints?: ToolConstraints;
-  tool_sequence?: string[];
-  tool_input_assertions?: ToolInputAssertion[];
-  response_assertions?: ResponseAssertion[];
-  agent_assertions?: AgentAssertion[];
-  agent_context?: AgentContext;
-}
-export type CheckResultStatus = 'passed' | 'failed' | 'not_evaluated';
+    tool_constraints?: ToolConstraints;
+    tool_sequence?: string[];
+    tool_input_assertions?: ToolInputAssertion[];
+    response_assertions?: ResponseAssertion[];
+    agent_assertions?: AgentAssertion[];
+    agent_context?: AgentContext;
+}
+export type CheckResultStatus = 'passed' | 'failed' | 'not_evaluated' | 'not_executed'
+/** @deprecated Persisted results used this name for capability-limited checks. */
+ | 'not_applicable';
 export interface CheckResult {
-  type: string;
-  label: string;
-  status: CheckResultStatus;
-  reason?: string;
-  metadata?: Record<string, unknown>;
+    type: string;
+    label: string;
+    status: CheckResultStatus;
+    reason?: string;
+    metadata?: Record<string, unknown>;
 }
 export interface CheckCounts {
-  passed: number;
-  failed: number;
-  not_evaluated: number;
-  total: number;
+    passed: number;
+    failed: number;
+    /** Checks intentionally outside the execution source's capabilities. */
+    not_evaluated: number;
+    /** Checks expected to run but skipped because execution ended early. */
+    not_executed?: number;
+    /** Legacy field retained when reading older result summaries. */
+    not_applicable?: number;
+    total: number;
 }
 export interface ExtractRule {
-  name: string;
-  from: 'final_text';
-  regex: string;
+    name: string;
+    from: 'final_text';
+    regex: string;
 }
 export interface ScenarioAttachment {
-  type: 'image' | 'document';
-  media_type: string;
-  data: string;
-  url?: string;
-  name?: string;
+    type: 'image' | 'document';
+    media_type: string;
+    data: string;
+    url?: string;
+    name?: string;
 }
 export interface SourceScenarioAttachment extends Omit<ScenarioAttachment, 'media_type' | 'data'> {
-  media_type?: string;
-  data?: string;
-  path?: string;
+    media_type?: string;
+    data?: string;
+    path?: string;
 }
 export interface Scenario {
-  id: string;
-  name?: string;
-  mcp_servers?: ServerListEntry[];
-  servers: string[];
-  prompt: string;
-  attachments?: ScenarioAttachment[];
-  eval?: EvalRules;
-  extract?: ExtractRule[];
+    id: string;
+    name?: string;
+    mcp_servers?: ServerListEntry[];
+    servers: string[];
+    prompt: string;
+    attachments?: ScenarioAttachment[];
+    eval?: EvalRules;
+    extract?: ExtractRule[];
 }
 export interface ScenarioRefEntry {
-  ref: string;
-  mcp_servers?: ServerListEntry[];
+    ref: string;
+    mcp_servers?: ServerListEntry[];
 }
 export interface SourceScenario extends Omit<Scenario, 'servers' | 'attachments'> {
-  servers?: string[];
-  attachments?: SourceScenarioAttachment[];
+    servers?: string[];
+    attachments?: SourceScenarioAttachment[];
 }
 export type ScenarioListEntry = SourceScenario | ScenarioRefEntry;
 export interface EvalConfig {
-  name?: string;
-  servers: Record<string, ServerConfig>;
-  agents: Record<string, AgentConfig>;
-  scenarios: Scenario[];
-  run_defaults?: {
-    selected_agents?: string[];
-  };
+    name?: string;
+    description?: string;
+    servers: Record<string, ServerConfig>;
+    agents: Record<string, AgentConfig>;
+    scenarios: Scenario[];
+    run_defaults?: {
+        selected_agents?: string[];
+    };
 }
 export interface SourceEvalConfig extends Omit<EvalConfig, 'scenarios' | 'agents' | 'servers'> {
-  servers?: ServerListEntry[];
-  agents: AgentListEntry[];
-  scenarios: ScenarioListEntry[];
+    servers?: ServerListEntry[];
+    agents: AgentListEntry[];
+    scenarios: ScenarioListEntry[];
 }
 export interface ExecutableScenario extends Scenario {
-  agent: string;
-  scenario_exec_id?: string;
+    agent: string;
+    scenario_exec_id?: string;
 }
 export interface ExecutableEvalConfig extends Omit<EvalConfig, 'scenarios'> {
-  scenarios: ExecutableScenario[];
+    scenarios: ExecutableScenario[];
 }
 export interface ToolDef {
-  name: string;
-  title?: string;
-  description?: string;
-  inputSchema?: unknown;
-  outputSchema?: unknown;
-  annotations?: {
+    name: string;
     title?: string;
-    readOnlyHint?: boolean;
-    idempotentHint?: boolean;
-    destructiveHint?: boolean;
-    openWorldHint?: boolean;
-    [key: string]: unknown;
-  };
+    description?: string;
+    inputSchema?: unknown;
+    outputSchema?: unknown;
+    annotations?: {
+        title?: string;
+        readOnlyHint?: boolean;
+        idempotentHint?: boolean;
+        destructiveHint?: boolean;
+        openWorldHint?: boolean;
+        [key: string]: unknown;
+    };
 }
 export interface ToolCall {
-  id?: string;
-  name: string;
-  arguments: Record<string, unknown> | unknown;
-  server?: string;
+    id?: string;
+    name: string;
+    arguments: Record<string, unknown> | unknown;
+    server?: string;
 }
 export interface LlmMessage {
-  role: 'system' | 'user' | 'assistant' | 'tool';
-  content: string;
-  attachments?: ScenarioAttachment[];
-  tool_call_id?: string;
-  name?: string;
-  tool_calls?: ToolCall[];
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    attachments?: ScenarioAttachment[];
+    tool_call_id?: string;
+    name?: string;
+    tool_calls?: ToolCall[];
 }
 export interface LlmResponse {
-  content?: string;
-  tool_calls?: ToolCall[];
-  raw?: unknown;
-  usage?: {
+    content?: string;
+    tool_calls?: ToolCall[];
+    raw?: unknown;
+    usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+        total_tokens?: number;
+    };
+}
+export interface TraceMessageUsage {
     input_tokens?: number;
     output_tokens?: number;
     total_tokens?: number;
-  };
-}
-export interface TraceMessageUsage {
-  input_tokens?: number;
-  output_tokens?: number;
-  total_tokens?: number;
 }
 export interface EstimatedTokens {
-  input: number;
-  output: number;
-  total: number;
-  method: 'js_tiktoken_estimate' | 'js_tiktoken_fallback';
-}
-export type TraceMessageContentBlock =
-  | {
-      type: 'text';
-      text: string;
-    }
-  | {
-      type: 'image';
-      media_type: string;
-      data: string;
-      name?: string;
-    }
-  | {
-      type: 'document';
-      media_type: string;
-      data: string;
-      name?: string;
-    }
-  | {
-      type: 'tool_use';
-      id: string;
-      name: string;
-      input: unknown;
-      server: string;
-      estimated_tokens?: EstimatedTokens;
-    }
-  | {
-      type: 'tool_result';
-      tool_use_id: string;
-      name: string;
-      content: Array<{
+    input: number;
+    output: number;
+    total: number;
+    method: 'js_tiktoken_estimate' | 'js_tiktoken_fallback';
+}
+export type TraceMessageContentBlock = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    media_type: string;
+    data: string;
+    name?: string;
+} | {
+    type: 'document';
+    media_type: string;
+    data: string;
+    name?: string;
+} | {
+    type: 'tool_use';
+    id: string;
+    name: string;
+    input: unknown;
+    server: string;
+    estimated_tokens?: EstimatedTokens;
+} | {
+    type: 'tool_result';
+    tool_use_id: string;
+    name: string;
+    content: Array<{
         type: 'text';
         text: string;
-      }>;
-      is_error: boolean;
-      duration_ms?: number;
-      ts_start?: string;
-      ts_end?: string;
-      server?: string;
-      estimated_tokens?: EstimatedTokens;
-    };
+    }>;
+    is_error: boolean;
+    duration_ms?: number;
+    ts_start?: string;
+    ts_end?: string;
+    server?: string;
+    estimated_tokens?: EstimatedTokens;
+};
 export interface TraceMessage {
-  role: 'user' | 'assistant' | 'tool';
-  ts?: string;
-  usage?: TraceMessageUsage;
-  content: TraceMessageContentBlock[];
+    role: 'user' | 'assistant' | 'tool';
+    ts?: string;
+    usage?: TraceMessageUsage;
+    content: TraceMessageContentBlock[];
 }
 export interface ScenarioRunTraceRecord {
-  type: 'scenario_run';
-  trace_version: 3;
-  run_index: number;
-  request_id?: string;
-  scenario_id: string;
-  agent: string;
-  provider: string;
-  model: string;
-  ts_start: string;
-  ts_end: string;
-  pass: boolean;
-  error?: string;
-  messages: TraceMessage[];
-  metrics?: {
-    tool_call_count: number;
-    total_tool_duration_ms: number;
-  };
+    type: 'scenario_run';
+    trace_version: 3;
+    run_index: number;
+    request_id?: string;
+    scenario_id: string;
+    agent: string;
+    provider: string;
+    model: string;
+    ts_start: string;
+    ts_end: string;
+    pass: boolean;
+    outcome?: RunOutcome;
+    error?: string;
+    messages: TraceMessage[];
+    metrics?: {
+        tool_call_count: number;
+        total_tool_duration_ms: number;
+    };
 }
+export type RunOutcome = 'passed' | 'failed' | 'incomplete' | 'error' | 'stopped';
+export type ExecutionSource = 'mcplab' | 'rover';
 export interface TraceFileLegacyMeta {
-  type: 'trace_meta';
-  trace_version: number;
-  run_id: string;
-  ts: string;
-}
-export type PersistedTraceRecord =
-  | ScenarioRunTraceRecord
-  | TraceFileLegacyMeta
-  | Record<string, unknown>;
+    type: 'trace_meta';
+    trace_version: number;
+    run_id: string;
+    ts: string;
+}
+export type PersistedTraceRecord = ScenarioRunTraceRecord | TraceFileLegacyMeta | Record<string, unknown>;
 export interface ScenarioRunResult {
-  run_index: number;
-  request_id?: string;
-  pass: boolean;
-  error?: string;
-  failures: string[];
-  check_results?: CheckResult[];
-  tool_calls: string[];
-  tool_call_count: number;
-  tool_sequence: string[];
-  tool_usage: Record<string, number>;
-  tool_durations_ms: number[];
-  run_duration_ms?: number;
-  final_text: string;
-  extracted: Record<string, string | number | boolean | null>;
+    run_index: number;
+    request_id?: string;
+    pass: boolean;
+    outcome?: RunOutcome;
+    error?: string;
+    stop_reason?: string;
+    failures: string[];
+    check_results?: CheckResult[];
+    tool_calls: string[];
+    tool_call_count: number;
+    tool_sequence: string[];
+    tool_usage: Record<string, number>;
+    tool_durations_ms: number[];
+    run_duration_ms?: number;
+    final_text: string;
+    extracted: Record<string, string | number | boolean | null>;
 }
 export interface ScenarioAggregate {
-  scenario_id: string;
-  scenario_name?: string;
-  agent: string;
-  provider?: string;
-  model?: string;
-  eval?: EvalRules;
-  tool_constraints_stats?: {
-    required: Record<string, number>;
-    forbidden: Record<string, number>;
-  };
-  runs: ScenarioRunResult[];
-  pass_rate: number;
-  distinct_sequences: Record<string, number>;
-  tool_usage_frequency: Record<string, number>;
-  extracted_values: Record<string, Record<string, number>>;
-  last_final_answer: string;
-}
-export type HealthMcpConnectionInfo =
-  | {
-      enabled: false;
-    }
-  | {
-      enabled: true;
-      transport: 'streamable-http';
-      host: string;
-      port: number;
-      path: string;
-      proxyUrl: string;
-      directUrl: string;
-      serverPackageVersion: string;
-      environment: {
+    scenario_id: string;
+    scenario_name?: string;
+    agent: string;
+    provider?: string;
+    model?: string;
+    eval?: EvalRules;
+    tool_constraints_stats?: {
+        required: Record<string, number>;
+        forbidden: Record<string, number>;
+    };
+    runs: ScenarioRunResult[];
+    pass_rate: number | null;
+    distinct_sequences: Record<string, number>;
+    tool_usage_frequency: Record<string, number>;
+    extracted_values: Record<string, Record<string, number>>;
+    last_final_answer: string;
+}
+export type HealthMcpConnectionInfo = {
+    enabled: false;
+} | {
+    enabled: true;
+    transport: 'streamable-http';
+    host: string;
+    port: number;
+    path: string;
+    proxyUrl: string;
+    directUrl: string;
+    serverPackageVersion: string;
+    environment: {
         MCP_HOST: string;
         MCP_PORT: string;
         MCP_PATH: string;
-      };
     };
+};
 export interface ResultsJson {
-  metadata: {
-    run_id: string;
-    timestamp: string;
-    run_note?: string;
-    git_commit?: string;
-    config_hash: string;
-    langsmith_trace_urls?: Record<string, string>;
-    config_path?: string;
-    config_name?: string;
-    rerun_agents?: string[];
-    rerun_scenario_ids?: string[];
-    rerun_server_override_all?: string[];
-    rerun_scenario_server_overrides?: Record<string, string[]>;
-    tool_tokens_total?: number | null;
-    total_duration_ms?: number;
-    total_tool_duration_ms?: number;
-    cli_version: string;
-    mcp_server_versions: Record<string, string | null>;
-  };
-  summary: {
-    total_scenarios: number;
-    total_runs: number;
-    pass_rate: number;
-    avg_tool_calls_per_run: number;
-    avg_tool_latency_ms: number | null;
-  };
-  scenarios: ScenarioAggregate[];
-}
-//# sourceMappingURL=types.d.ts.map
+    metadata: {
+        run_id: string;
+        timestamp: string;
+        run_note?: string;
+        git_commit?: string;
+        config_hash: string;
+        langsmith_trace_urls?: Record<string, string>;
+        config_path?: string;
+        config_name?: string;
+        rerun_agents?: string[];
+        rerun_scenario_ids?: string[];
+        rerun_server_override_all?: string[];
+        rerun_scenario_server_overrides?: Record<string, string[]>;
+        rerun_new_conversation_before_start?: boolean;
+        rerun_new_conversation_between_scenarios?: boolean;
+        tool_tokens_total?: number | null;
+        total_duration_ms?: number;
+        total_tool_duration_ms?: number;
+        cli_version: string;
+        mcp_server_versions: Record<string, string | null>;
+        execution_source?: ExecutionSource;
+        execution_client?: string;
+        /** @deprecated Read legacy results only. New evaluations use evaluation_run_id. */
+        evaluation_group_id?: string;
+        evaluation_run_id?: string;
+        /** @deprecated Read legacy parent-result projections only. */
+        child_run_ids?: string[];
+        execution_status?: 'stopped';
+    };
+    summary: {
+        total_scenarios: number;
+        total_runs: number;
+        pass_rate: number | null;
+        avg_tool_calls_per_run: number;
+        avg_tool_latency_ms: number | null;
+        outcomes?: Partial<Record<RunOutcome, number>>;
+    };
+    scenarios: ScenarioAggregate[];
+}
+//# sourceMappingURL=types.d.ts.map