@inspectr/mcplab-core 1.22.2 → 1.24.0

package/dist/runner.d.ts

@@ -1,138 +1,87 @@
-import type {
-  AgentAssertion,
-  AgentConfig,
-  AgentJudgeContext,
-  ExecutableEvalConfig,
-  ResultsJson,
-  ExecutableScenario
-} from './types.js';
+import type { ExecutableEvalConfig, ResultsJson, ExecutableScenario, RunOutcome, WorkspaceEvaluation } from './types.js';
 import { type AgentRunProgressEvent } from './agent.js';
 import { type TraceExporter } from './langsmith-tracing.js';
-export interface RunOptions {
-  runsPerScenario: number;
-  scenarioId?: string;
-  runNote?: string;
-  configHash: string;
-  gitCommit?: string;
-  cliVersion: string;
-  runsDir?: string;
-  cwd?: string;
-  mcpServerAuthHeaders?: Record<string, Record<string, string>>;
-  oauthTokens?: Record<string, string>;
-  resolveMcpServerAuthHeaders?: (
-    serverNames: string[],
-    options?: {
-      signal?: AbortSignal;
-    }
-  ) => Promise<Record<string, Record<string, string>>>;
-  evaluationJudge?: {
-    name: string;
-    agent: AgentConfig;
-  };
-  signal?: AbortSignal;
-  onProgress?: (event: RunProgressEvent) => void | Promise<void>;
-  traceExporter?: TraceExporter;
+export interface RunOptions extends WorkspaceEvaluation {
+    runsPerScenario: number;
+    scenarioId?: string;
+    runNote?: string;
+    configHash: string;
+    gitCommit?: string;
+    cliVersion: string;
+    runsDir?: string;
+    cwd?: string;
+    mcpServerAuthHeaders?: Record<string, Record<string, string>>;
+    oauthTokens?: Record<string, string>;
+    resolveMcpServerAuthHeaders?: (serverNames: string[], options?: {
+        signal?: AbortSignal;
+    }) => Promise<Record<string, Record<string, string>>>;
+    signal?: AbortSignal;
+    /** Optional signal per executable scenario-agent pair. A child abort skips only that pair. */
+    scenarioSignal?: (scenario: ExecutableScenario, runIndex: number) => AbortSignal | undefined;
+    onProgress?: (event: RunProgressEvent) => void | Promise<void>;
+    traceExporter?: TraceExporter;
+    /** Skip standard result projections when a caller owns shared persistence. */
+    persistArtifacts?: boolean;
 }
-export type RunProgressEvent =
-  | {
-      type: 'run_started';
-      runId: string;
-      totalScenarioRuns: number;
-      runsPerScenario: number;
-    }
-  | {
-      type: 'mcp_connect_started';
-      serverCount: number;
-      serverNames: string[];
-    }
-  | {
-      type: 'mcp_connect_finished';
-      serverCount: number;
-      serverNames: string[];
-    }
-  | {
-      type: 'scenario_run_started';
-      scenarioId: string;
-      agentName: string;
-      scenarioRunIndex: number;
-      totalScenarioRuns: number;
-      runIndex: number;
-      runsPerScenario: number;
-    }
-  | {
-      type: 'scenario_run_finished';
-      scenarioId: string;
-      agentName: string;
-      scenarioRunIndex: number;
-      totalScenarioRuns: number;
-      runIndex: number;
-      runsPerScenario: number;
-      pass: boolean;
-      toolCallCount: number;
-    }
-  | {
-      type: 'agent_progress';
-      scenarioRunIndex: number;
-      totalScenarioRuns: number;
-      event: AgentRunProgressEvent;
-    }
-  | {
-      type: 'run_finished';
-      runId: string;
-      totalScenarioRuns: number;
-    };
-export declare function buildMcpServerAuthHeaders(
-  options: Pick<RunOptions, 'mcpServerAuthHeaders' | 'oauthTokens'>
-): Record<string, Record<string, string>>;
-export declare function runAll(
-  config: ExecutableEvalConfig,
-  options: RunOptions
-): Promise<{
-  runDir: string;
-  results: ResultsJson;
-}>;
-export declare function buildJudgeBatchPayload(
-  finalText: string,
-  assertions: AgentAssertion[],
-  context?: AgentJudgeContext
-): {
-  checks: {
-    id: string;
-    label: string;
-    prompt: string;
-  }[];
-  context?: AgentJudgeContext | undefined;
-  final_answer: string;
+export type RunProgressEvent = {
+    type: 'run_started';
+    runId: string;
+    totalScenarioRuns: number;
+    runsPerScenario: number;
+} | {
+    type: 'mcp_connect_started';
+    serverCount: number;
+    serverNames: string[];
+} | {
+    type: 'mcp_connect_finished';
+    serverCount: number;
+    serverNames: string[];
+} | {
+    type: 'scenario_run_started';
+    scenarioId: string;
+    agentName: string;
+    scenarioRunIndex: number;
+    totalScenarioRuns: number;
+    runIndex: number;
+    runsPerScenario: number;
+} | {
+    type: 'scenario_run_finished';
+    scenarioId: string;
+    agentName: string;
+    scenarioRunIndex: number;
+    totalScenarioRuns: number;
+    runIndex: number;
+    runsPerScenario: number;
+    pass: boolean;
+    outcome?: RunOutcome;
+    toolCallCount: number;
+} | {
+    type: 'agent_progress';
+    scenarioRunIndex: number;
+    totalScenarioRuns: number;
+    event: AgentRunProgressEvent;
+} | {
+    type: 'run_finished';
+    runId: string;
+    totalScenarioRuns: number;
 };
-type JudgeBatchPayload = ReturnType<typeof buildJudgeBatchPayload>;
-export declare function mapJudgeBatchResults(params: {
-  judgeName: string;
-  judgeAgent: AgentConfig;
-  batch: JudgeBatchPayload;
-  raw: string;
-}): Array<{
-  label: string;
-  pass: boolean;
-  reason: string;
-  metadata?: Record<string, unknown>;
+export declare function buildMcpServerAuthHeaders(options: Pick<RunOptions, 'mcpServerAuthHeaders' | 'oauthTokens'>): Record<string, Record<string, string>>;
+export declare function runAll(config: ExecutableEvalConfig, options: RunOptions): Promise<{
+    runDir: string;
+    results: ResultsJson;
 }>;
-export declare function extractJudgeJson(raw: string): string;
-export declare function filterScenariosForRun(
-  scenarios: ExecutableScenario[],
-  scenarioId?: string
-): ExecutableScenario[];
+export declare function filterScenariosForRun(scenarios: ExecutableScenario[], scenarioId?: string): ExecutableScenario[];
 export declare function createRunId(now?: Date): string;
 export declare function buildScenarioRequestId(params: {
-  runId: string;
-  scenarioId?: string;
-  agentName?: string;
-  scenarioExecId?: string;
-  runIndex: number;
+    runId: string;
+    scenarioId?: string;
+    agentName?: string;
+    scenarioExecId?: string;
+    runIndex: number;
 }): string;
 export declare function buildFallbackScenarioRequestId(params: {
-  runId: string;
-  runIndex: number;
-  scenarioAgent?: string;
+    runId: string;
+    runIndex: number;
+    scenarioAgent?: string;
 }): string;
-export {};
-//# sourceMappingURL=runner.d.ts.map
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,cAAc,EACd,WAAW,EACX,iBAAiB,EACjB,oBAAoB,EAEpB,WAAW,EACX,kBAAkB,EAEnB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,YAAY,CAAC;AASpB,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,2BAA2B,CAAC,EAAE,CAC5B,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,KAC/B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrD,eAAe,CAAC,EAAE;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,WAAW,CAAC;KACpB,CAAC;IACF,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;CACzB,GACD;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAA;CAAE,GAC3E;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAA;CAAE,GAC5E;IACE,IAAI,EAAE,sBAAsB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB,GACD;IACE,IAAI,EAAE,uBAAuB,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,OAAO,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,gBAAgB,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,qBAAqB,CAAC;CAC9B,GACD;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,sBAAsB,GAAG,aAAa,CAAC,GAChE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAQxC;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,oBAAoB,EAC5B,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,WAAW,CAAA;CAAE,CAAC,CA0WnD;AAED,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,cAAc,EAAE,EAC5B,OAAO,CAAC,EAAE,iBAAiB;;;;;;;;EAW5B;AAED,KAAK,iBAAiB,GAAG,UAAU,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEnE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;IACxB,KAAK,EAAE,iBAAiB,CAAC;IACzB,GAAG,EAAE,MAAM,CAAC;CACb,GAAG,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAAC,CA+D9F;AAsDD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAapD;AAmCD,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,kBAAkB,EAAE,EAC/B,UAAU,CAAC,EAAE,MAAM,GAClB,kBAAkB,EAAE,CAGtB;AAKD,wBAAgB,WAAW,CAAC,GAAG,GAAE,IAAiB,GAAG,MAAM,CAsB1D;AAID,wBAAgB,sBAAsB,CAAC,MAAM,EAAE;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,MAAM,CAUT;AAED,wBAAgB,8BAA8B,CAAC,MAAM,EAAE;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,MAAM,CAOT"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,oBAAoB,EAEpB,WAAW,EACX,kBAAkB,EAElB,UAAU,EACV,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAKpB,OAAO,EAAoB,KAAK,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAO1E,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,UAAW,SAAQ,mBAAmB;IACrD,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,2BAA2B,CAAC,EAAE,CAC5B,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,KAC/B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,8FAA8F;IAC9F,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,KAAK,WAAW,GAAG,SAAS,CAAC;IAC7F,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;CACzB,GACD;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAA;CAAE,GAC3E;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAA;CAAE,GAC5E;IACE,IAAI,EAAE,sBAAsB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB,GACD;IACE,IAAI,EAAE,uBAAuB,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,gBAAgB,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,qBAAqB,CAAC;CAC9B,GACD;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,sBAAsB,GAAG,aAAa,CAAC,GAChE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAQxC;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,oBAAoB,EAC5B,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,WAAW,CAAA;CAAE,CAAC,CAyYnD;AAED,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,kBAAkB,EAAE,EAC/B,UAAU,CAAC,EAAE,MAAM,GAClB,kBAAkB,EAAE,CAGtB;AAKD,wBAAgB,WAAW,CAAC,GAAG,GAAE,IAAiB,GAAG,MAAM,CAsB1D;AAID,wBAAgB,sBAAsB,CAAC,MAAM,EAAE;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,MAAM,CAUT;AAED,wBAAgB,8BAA8B,CAAC,MAAM,EAAE;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,MAAM,CAOT"}