@gleanwork/mcp-server-tester 2.0.0-beta.0 → 2.0.0-beta.1

package/dist/{index-aX_tCfTD.d.cts → index-CwFHKIoN.d.cts}

@@ -7,6 +7,22 @@ import { OAuthClientMetadata, OAuthClientInformationFull, OAuthTokens } from '@m
 import * as oauth from 'oauth4webapi';
 import * as playwright_test from 'playwright/test';
 
+/** Sanitized host evidence, not an evaluation verdict or raw process log. */
+interface HostDiagnostics {
+    failureKind?: 'startup' | 'timeout' | 'process' | 'output';
+    claudeStartup?: {
+        status: 'ready' | 'failed' | 'missing';
+        elapsedMs: number;
+        model?: string;
+        version?: string;
+        servers: Array<{
+            name: string;
+            status: 'connected' | 'pending' | 'failed' | 'needs-auth' | 'disabled' | 'unknown';
+            tools: string[];
+        }>;
+    };
+}
+
 /**
  * Validator Types
  *
@@ -433,6 +449,14 @@ declare function isStdioConfig(config: MCPConfig): config is StdioMCPConfig;
  */
 declare function isHttpConfig(config: MCPConfig): config is HttpMCPConfig;
 
+/** Native Cowork configuration only; this does not authorize the eval runner to
+ * execute writes. Unset/false preserves Cowork's existing approval behavior.
+ */
+declare const CoworkSetupConfigSchema: z.ZodObject<{
+    approveWriteTools: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>;
+type CoworkSetupConfig = z.infer<typeof CoworkSetupConfigSchema>;
+
 /** A tagged configuration block resolved by a public registry. */
 interface TaggedConfig {
     type: string;
@@ -461,6 +485,7 @@ interface EvalArm {
     scenarioTemplate?: string;
     metrics?: ExtensionConfig[];
     judges?: ExtensionConfig[];
+    coworkSetup?: CoworkSetupConfig;
 }
 /** A complete, organization-neutral evaluation manifest. */
 interface EvalManifest {
@@ -474,6 +499,7 @@ interface EvalManifest {
     arms?: EvalArm[];
     metrics?: ExtensionConfig[];
     judges?: ExtensionConfig[];
+    coworkSetup?: CoworkSetupConfig;
     results?: {
         store: ExtensionConfig;
     };
@@ -651,6 +677,9 @@ declare const EvalManifestSchema: z.ZodObject<{
         judges: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             type: z.ZodString;
         }, z.core.$loose>]>>>;
+        coworkSetup: z.ZodOptional<z.ZodObject<{
+            approveWriteTools: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>>;
     }, z.core.$strict>>>;
     metrics: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         type: z.ZodString;
@@ -658,6 +687,9 @@ declare const EvalManifestSchema: z.ZodObject<{
     judges: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         type: z.ZodString;
     }, z.core.$loose>]>>>;
+    coworkSetup: z.ZodOptional<z.ZodObject<{
+        approveWriteTools: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>;
     results: z.ZodOptional<z.ZodObject<{
         store: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             type: z.ZodString;
@@ -864,6 +896,8 @@ type CLIOutputFormat = 'stream-json' | 'json';
  * ```
  */
 interface CLIConfig {
+    /** Claude Code only: validate these MCP servers in its startup stream. */
+    claudeMcpServers?: string[];
     /** Child-process-only environment overrides. Undefined removes an inherited key. */
     env?: Record<string, string | undefined>;
     /**
@@ -1012,11 +1046,15 @@ interface LLMToolCall {
     id?: string;
     /** Tool result text, when the host surfaces it (paired to this call) */
     output?: string;
+    /** Explicit tool-result error status; absent when not observed. */
+    isError?: boolean;
 }
 /**
  * Result from an MCP host simulation
  */
 interface MCPHostSimulationResult {
+    /** Sanitized startup evidence, retained even when execution fails. */
+    diagnostics?: HostDiagnostics;
     /** Whether the simulation succeeded */
     success: boolean;
     /** Tool calls made by the LLM */
@@ -1752,6 +1790,7 @@ declare const EvalCaseSchema: z.ZodObject<{
                 "stream-json": "stream-json";
                 json: "json";
             }>>;
+            claudeMcpServers: z.ZodOptional<z.ZodArray<z.ZodString>>;
             timeout: z.ZodOptional<z.ZodNumber>;
         }, z.core.$strip>>;
         mcpServers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
@@ -2068,6 +2107,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
                     "stream-json": "stream-json";
                     json: "json";
                 }>>;
+                claudeMcpServers: z.ZodOptional<z.ZodArray<z.ZodString>>;
                 timeout: z.ZodOptional<z.ZodNumber>;
             }, z.core.$strip>>;
             mcpServers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
@@ -2626,10 +2666,14 @@ interface IterationResult {
             name: string;
         }>;
     };
+    /** Sanitized host evidence for this specific attempt. */
+    hostDiagnostics?: HostDiagnostics;
     /** Evidence level retained even when raw responses are redacted. */
     hostEvidence?: HostEvidence;
     /** Token usage from mcp_host LLM simulation in this iteration */
     hostUsage?: UsageMetrics;
+    /** Native numeric host measurements, retained after response redaction. */
+    hostTelemetry?: Record<string, unknown>;
     /** External host metadata for this iteration */
     externalHost?: ExternalHostMetadata;
 }
@@ -2809,6 +2853,8 @@ interface EvalCaseResult {
             name: string;
         }>;
     };
+    /** Sanitized host evidence; each iteration retains its own diagnostics. */
+    hostDiagnostics?: HostDiagnostics;
     /** Evidence level retained in persisted comparisons after response redaction. */
     hostEvidence?: HostEvidence;
     /**
@@ -2816,6 +2862,8 @@ interface EvalCaseResult {
      * Summed across all iterations. Only populated for mcp_host mode cases.
      */
     hostUsage?: UsageMetrics;
+    /** Native single-iteration measurements; multi-iteration values live in iterationResults. */
+    hostTelemetry?: Record<string, unknown>;
     /**
      * External host trace and evidence metadata.
      * Populated for external_host mode cases.
@@ -2943,6 +2991,9 @@ interface EvalExecutionResult {
     response: unknown;
     error?: string;
     hostUsage?: UsageMetrics;
+    hostTelemetry?: Record<string, unknown>;
+    /** Host time already spent outside the current evaluation timer. */
+    preExecutionDurationMs?: number;
     evidence?: HostEvidence;
 }
 
@@ -3306,14 +3357,29 @@ interface HostEvent {
     server?: string;
     arguments?: Record<string, unknown>;
     output?: string;
+    /** Explicit tool-result error status, absent when not observed. */
+    isError?: boolean;
+    rawName?: string;
     id?: string;
 }
 /** One execution trace. Hosts never return evaluation verdicts. */
 interface HostRunResult {
+    diagnostics?: HostDiagnostics;
     finalText: string;
     events: HostEvent[];
     error?: string;
     usage?: UsageMetrics;
+    /** Native and driver observations remain separately scoped; UI actions are not LLM usage. */
+    telemetry?: Record<string, unknown>;
+    /** Per-request wall time, excluding shared batch setup and cleanup. */
+    durationMs?: number;
+    llmDurationMs?: number;
+}
+interface HostBatchRequest {
+    caseId: string;
+    iteration: number;
+    input: HostRunInput;
+    config: HostConfig;
 }
 /** Public host extension point. */
 interface HostDefinition {
@@ -3322,6 +3388,8 @@ interface HostDefinition {
     createConfig?(options?: Record<string, unknown>): MCPHostConfig;
     /** Missing evidence declarations are treated as unverified. */
     readonly evidence?: HostEvidence;
+    /** Ordered traces for all selected iterations. The framework owns verdicts. */
+    runBatch?(requests: HostBatchRequest[], context: HostRunContext): Promise<HostRunResult[]>;
     run?(input: HostRunInput, config: HostConfig, context: HostRunContext): Promise<HostRunResult>;
 }
 /** Values emitted by a metric for one evaluation case. */
@@ -5310,6 +5378,7 @@ declare function runConformanceChecks(mcp: MCPFixtureApi, options?: MCPConforman
  * - 'none': No authentication
  */
 type AuthType = 'oauth' | 'api-token' | 'none';
+
 /**
  * Source of test results
  *
@@ -5370,4 +5439,4 @@ type ExpectationResultMap = Partial<Record<ExpectationType, EvalExpectationResul
  */
 type ExpectationBreakdown = Partial<Record<ExpectationType, number>>;
 
-export { type CompareEvalRunsOptions as $, type AuthType as A, type HostDriverConfig as B, type ExternalHostCapabilitiesConfig as C, type DatasetSource as D, type ExtensionConfig as E, type JudgeConfig as F, type Judge as G, type HostDefinition as H, BUILT_IN_RUBRICS as I, type JudgeMatcherOptions as J, type BuiltInRubric as K, type LLMProvider as L, type MCPFixtureApi as M, type BuiltInSanitizer as N, type OAuthSetupConfig as O, type PatternValidatorOptions as P, type CLIConfig as Q, type RubricSpec as R, type SchemaValidatorOptions as S, type TextValidatorOptions as T, CLIOAuthClient as U, type ValidationResult as V, type CLIOAuthClientConfig as W, type CLIOAuthResult as X, type CLIOutputFormat as Y, type CaseComparisonResult as Z, type ClientCredentialsConfig as _, type MCPHostConfig as a, type JudgeExpectConfig as a$, type ComparisonOutcome as a0, type ContentBlock as a1, type CreateMCPClientOptions as a2, type CustomJudgeExecutor as a3, type CustomJudgeResult as a4, type DatasetSourceContext as a5, DiscoveryError as a6, ENV_VAR_NAMES as a7, type EvalArm as a8, type EvalCase as a9, type ExpectationType as aA, type ExperimentMetric as aB, type ExternalHostCapabilityBinding as aC, type ExternalHostCapabilityContext as aD, type ExternalHostFailureKind as aE, type ExternalHostMetadata as aF, type ExternalHostSession as aG, type ExternalHostSimulationResult as aH, type ExternalHostType as aI, type FieldRemovalSanitizer as aJ, FileEvalResultStore as aK, type FileEvalResultStoreConfig as aL, GCSEvalResultStore as aM, type GCSEvalResultStoreConfig as aN, type HostArtifact as aO, type HostCapability as aP, type HostConfig as aQ, type HostConfigPatch as aR, type HostEvent as aS, type HostEvidence as aT, type HostRunContext$1 as aU, type HostRunInput as aV, type HostRunOptions as aW, type HostRunResult as aX, type HostType as aY, type HttpMCPConfig as aZ, type IterationResult as a_, type EvalCaseComparison as aa, type EvalCaseComparisonOutcome as ab, type EvalCaseRequest as ac, EvalCaseSchema as ad, type EvalContext as ae, EvalDatasetSchema as af, type EvalExpectBlock as ag, type EvalExpectationResult as ah, type EvalManifestInput as ai, EvalManifestSchema as aj, type EvalMode as ak, type EvalResultStore as al, type EvalResultStoreConfig as am, type EvalResultStoreLike as an, type EvalRunComparisonLabels as ao, type EvalRunComparisonResult as ap, type EvalRunMetadata as aq, type EvalRunnerOptions as ar, type EvalSummaryGenerator as as, type EvaluationArmResult as at, type HostRunContext as au, type EvaluationSuiteOptions as av, type EvaluationSuiteResult as aw, type EvidenceSource as ax, type ExpectationBreakdown as ay, type ExpectationResultMap as az, type MCPHostSimulationResult as b, type TraceSource as b$, type JudgeResult as b0, type JudgeValidatorConfig as b1, type LLMToolCall as b2, type ListStoredArtifactsOptions as b3, type LoadDatasetOptions as b4, type MCPAuthConfig as b5, type MCPAuthFixtures as b6, type MCPClientCredentialsConfig as b7, MCPConfigSchema as b8, type MCPConformanceCheck as b9, type RunSummary as bA, type RunTelemetry as bB, type SaveBaselineOptions as bC, type SaveEvalRunComparisonOptions as bD, type SaveServerComparisonOptions as bE, type SchemaRegistry as bF, type SerializedEvalDataset as bG, type ServerComparisonOptions as bH, type ServerComparisonResult as bI, SnapshotSanitizers as bJ, type StdioMCPConfig as bK, type StoredArtifactKind as bL, type StoredArtifactSummary as bM, type StoredClientInfo as bN, type StoredEvalArtifact as bO, type StoredEvalArtifactMetadata as bP, type StoredEvalResultLoadOptions as bQ, type StoredEvalResultRef as bR, type StoredEvalResultSaveOptions as bS, type StoredEvalRunRef as bT, type StoredOAuthState as bU, type StoredServerMetadata as bV, type StoredTokens as bW, type TaggedConfig as bX, type TokenResult as bY, type ToolMetadataOverride as bZ, type ToolOverrideVariant as b_, type MCPConformanceOptions as ba, type MCPConformanceRaw as bb, type MCPConformanceResult as bc, type MCPConformanceResultData as bd, type MCPEvalData as be, type MCPEvalHistoricalSummary as bf, type MCPEvalReporterConfig as bg, type MCPEvalRunData as bh, type MCPFixtureOptions as bi, type MCPHostCapabilities as bj, type MCPHostSimulator as bk, type MCPOAuthConfig as bl, type MCPServerCapabilitiesData as bm, MCP_PROTOCOL_VERSION as bn, type MetricKind as bo, type NormalizedToolResponse as bp, type ObservationConfidence as bq, PlaywrightOAuthClientProvider as br, type PlaywrightOAuthClientProviderConfig as bs, type PredicateResult as bt, type ProposeVariantsContext as bu, type ProtectedResourceDiscoveryResult as bv, type ProtectedResourceMetadata as bw, type ProviderKind as bx, type RegexSanitizer as by, type ResultSource as bz, type SizeValidatorOptions as c, type UsageMetrics as c0, type VariantCandidateResult as c1, type VariantExperimentOptions as c2, type VariantExperimentReason as c3, type VariantExperimentResult as c4, type VariantExperimentRound as c5, type VariantImprovementProposal as c6, type VariantRecommendation as c7, closeMCPClient as c8, compareEvalRuns as c9, refreshAccessToken as cA, registerJudge as cB, resolveDatasetPaths as cC, resolveEvalResultStore as cD, resolveRubric as cE, runConformanceChecks as cF, runEvalCase as cG, runEvalDataset as cH, runServerComparison as cI, runVariantExperiment as cJ, saveBaseline as cK, saveEvalRunComparison as cL, saveServerComparison as cM, validateEvalCase as cN, validateEvalDataset as cO, validateJudge as cP, validateMCPConfig as cQ, validateToolCallCount as cR, validateToolCalls as cS, createDefaultArtifactId as ca, createEvalResultStore as cb, createMCPClientForConfig as cc, createMCPFixture as cd, createStoredEvalArtifact as ce, defaultEnvironmentMetadata as cf, discoverAuthorizationServer as cg, discoverProtectedResource as ch, extractText as ci, hasValidTokens as cj, injectTokens as ck, isBuiltInRubric as cl, isEvalResultStore as cm, isHttpConfig as cn, isStdioConfig as co, loadBaseline as cp, loadEvalDataset as cq, loadEvalDatasetFromObject as cr, loadEvalManifest as cs, loadEvalManifestFromObject as ct, loadStoredEvalRunnerResult as cu, loadTokens as cv, loadTokensFromEnv as cw, test as cx, normalizeToolResponse as cy, performClientCredentialsFlow as cz, type SnapshotSanitizer as d, type ToolPredicate as e, type ToolCallExpectation as f, type ToolCallCountOptions as g, type JudgeDefinition as h, type MetricDefinition as i, type ResultStoreDefinition as j, type EvalManifest as k, type EvalCaseResult as l, type MetricValue as m, type ResolvedMetric as n, type EvalDataset as o, type MCPConfig as p, type DatasetConfig as q, type EvalRunnerResult as r, type EvaluationSummary as s, type EvaluationBatchItem as t, type EvaluationBatchOptions as u, type EvaluationBatchResult as v, type ExternalHostConfig as w, type ExternalHostRunResult as x, type ExternalHostCapabilityImplementation as y, type HostDriverId as z };
+export { type CompareEvalRunsOptions as $, type AuthType as A, type HostDriverConfig as B, type ExternalHostCapabilitiesConfig as C, type DatasetSource as D, type ExtensionConfig as E, type JudgeConfig as F, type Judge as G, type HostDefinition as H, BUILT_IN_RUBRICS as I, type JudgeMatcherOptions as J, type BuiltInRubric as K, type LLMProvider as L, type MCPFixtureApi as M, type BuiltInSanitizer as N, type OAuthSetupConfig as O, type PatternValidatorOptions as P, type CLIConfig as Q, type RubricSpec as R, type SchemaValidatorOptions as S, type TextValidatorOptions as T, CLIOAuthClient as U, type ValidationResult as V, type CLIOAuthClientConfig as W, type CLIOAuthResult as X, type CLIOutputFormat as Y, type CaseComparisonResult as Z, type ClientCredentialsConfig as _, type MCPHostConfig as a, type HttpMCPConfig as a$, type ComparisonOutcome as a0, type ContentBlock as a1, type CreateMCPClientOptions as a2, type CustomJudgeExecutor as a3, type CustomJudgeResult as a4, type DatasetSourceContext as a5, DiscoveryError as a6, ENV_VAR_NAMES as a7, type EvalArm as a8, type EvalCase as a9, type ExpectationType as aA, type ExperimentMetric as aB, type ExternalHostCapabilityBinding as aC, type ExternalHostCapabilityContext as aD, type ExternalHostFailureKind as aE, type ExternalHostMetadata as aF, type ExternalHostSession as aG, type ExternalHostSimulationResult as aH, type ExternalHostType as aI, type FieldRemovalSanitizer as aJ, FileEvalResultStore as aK, type FileEvalResultStoreConfig as aL, GCSEvalResultStore as aM, type GCSEvalResultStoreConfig as aN, type HostArtifact as aO, type HostBatchRequest as aP, type HostCapability as aQ, type HostConfig as aR, type HostConfigPatch as aS, type HostDiagnostics as aT, type HostEvent as aU, type HostEvidence as aV, type HostRunContext$1 as aW, type HostRunInput as aX, type HostRunOptions as aY, type HostRunResult as aZ, type HostType as a_, type EvalCaseComparison as aa, type EvalCaseComparisonOutcome as ab, type EvalCaseRequest as ac, EvalCaseSchema as ad, type EvalContext as ae, EvalDatasetSchema as af, type EvalExpectBlock as ag, type EvalExpectationResult as ah, type EvalManifestInput as ai, EvalManifestSchema as aj, type EvalMode as ak, type EvalResultStore as al, type EvalResultStoreConfig as am, type EvalResultStoreLike as an, type EvalRunComparisonLabels as ao, type EvalRunComparisonResult as ap, type EvalRunMetadata as aq, type EvalRunnerOptions as ar, type EvalSummaryGenerator as as, type EvaluationArmResult as at, type HostRunContext as au, type EvaluationSuiteOptions as av, type EvaluationSuiteResult as aw, type EvidenceSource as ax, type ExpectationBreakdown as ay, type ExpectationResultMap as az, type MCPHostSimulationResult as b, type ToolMetadataOverride as b$, type IterationResult as b0, type JudgeExpectConfig as b1, type JudgeResult as b2, type JudgeValidatorConfig as b3, type LLMToolCall as b4, type ListStoredArtifactsOptions as b5, type LoadDatasetOptions as b6, type MCPAuthConfig as b7, type MCPAuthFixtures as b8, type MCPClientCredentialsConfig as b9, type RegexSanitizer as bA, type ResultSource as bB, type RunSummary as bC, type RunTelemetry as bD, type SaveBaselineOptions as bE, type SaveEvalRunComparisonOptions as bF, type SaveServerComparisonOptions as bG, type SchemaRegistry as bH, type SerializedEvalDataset as bI, type ServerComparisonOptions as bJ, type ServerComparisonResult as bK, SnapshotSanitizers as bL, type StdioMCPConfig as bM, type StoredArtifactKind as bN, type StoredArtifactSummary as bO, type StoredClientInfo as bP, type StoredEvalArtifact as bQ, type StoredEvalArtifactMetadata as bR, type StoredEvalResultLoadOptions as bS, type StoredEvalResultRef as bT, type StoredEvalResultSaveOptions as bU, type StoredEvalRunRef as bV, type StoredOAuthState as bW, type StoredServerMetadata as bX, type StoredTokens as bY, type TaggedConfig as bZ, type TokenResult as b_, MCPConfigSchema as ba, type MCPConformanceCheck as bb, type MCPConformanceOptions as bc, type MCPConformanceRaw as bd, type MCPConformanceResult as be, type MCPConformanceResultData as bf, type MCPEvalData as bg, type MCPEvalHistoricalSummary as bh, type MCPEvalReporterConfig as bi, type MCPEvalRunData as bj, type MCPFixtureOptions as bk, type MCPHostCapabilities as bl, type MCPHostSimulator as bm, type MCPOAuthConfig as bn, type MCPServerCapabilitiesData as bo, MCP_PROTOCOL_VERSION as bp, type MetricKind as bq, type NormalizedToolResponse as br, type ObservationConfidence as bs, PlaywrightOAuthClientProvider as bt, type PlaywrightOAuthClientProviderConfig as bu, type PredicateResult as bv, type ProposeVariantsContext as bw, type ProtectedResourceDiscoveryResult as bx, type ProtectedResourceMetadata as by, type ProviderKind as bz, type SizeValidatorOptions as c, type ToolOverrideVariant as c0, type TraceSource as c1, type UsageMetrics as c2, type VariantCandidateResult as c3, type VariantExperimentOptions as c4, type VariantExperimentReason as c5, type VariantExperimentResult as c6, type VariantExperimentRound as c7, type VariantImprovementProposal as c8, type VariantRecommendation as c9, normalizeToolResponse as cA, performClientCredentialsFlow as cB, refreshAccessToken as cC, registerJudge as cD, resolveDatasetPaths as cE, resolveEvalResultStore as cF, resolveRubric as cG, runConformanceChecks as cH, runEvalCase as cI, runEvalDataset as cJ, runServerComparison as cK, runVariantExperiment as cL, saveBaseline as cM, saveEvalRunComparison as cN, saveServerComparison as cO, validateEvalCase as cP, validateEvalDataset as cQ, validateJudge as cR, validateMCPConfig as cS, validateToolCallCount as cT, validateToolCalls as cU, closeMCPClient as ca, compareEvalRuns as cb, createDefaultArtifactId as cc, createEvalResultStore as cd, createMCPClientForConfig as ce, createMCPFixture as cf, createStoredEvalArtifact as cg, defaultEnvironmentMetadata as ch, discoverAuthorizationServer as ci, discoverProtectedResource as cj, extractText as ck, hasValidTokens as cl, injectTokens as cm, isBuiltInRubric as cn, isEvalResultStore as co, isHttpConfig as cp, isStdioConfig as cq, loadBaseline as cr, loadEvalDataset as cs, loadEvalDatasetFromObject as ct, loadEvalManifest as cu, loadEvalManifestFromObject as cv, loadStoredEvalRunnerResult as cw, loadTokens as cx, loadTokensFromEnv as cy, test as cz, type SnapshotSanitizer as d, type ToolPredicate as e, type ToolCallExpectation as f, type ToolCallCountOptions as g, type JudgeDefinition as h, type MetricDefinition as i, type ResultStoreDefinition as j, type EvalManifest as k, type EvalCaseResult as l, type MetricValue as m, type ResolvedMetric as n, type EvalDataset as o, type MCPConfig as p, type DatasetConfig as q, type EvalRunnerResult as r, type EvaluationSummary as s, type EvaluationBatchItem as t, type EvaluationBatchOptions as u, type EvaluationBatchResult as v, type ExternalHostConfig as w, type ExternalHostRunResult as x, type ExternalHostCapabilityImplementation as y, type HostDriverId as z };

package/dist/{index-aX_tCfTD.d.ts → index-CwFHKIoN.d.ts}

@@ -7,6 +7,22 @@ import { OAuthClientMetadata, OAuthClientInformationFull, OAuthTokens } from '@m
 import * as oauth from 'oauth4webapi';
 import * as playwright_test from 'playwright/test';
 
+/** Sanitized host evidence, not an evaluation verdict or raw process log. */
+interface HostDiagnostics {
+    failureKind?: 'startup' | 'timeout' | 'process' | 'output';
+    claudeStartup?: {
+        status: 'ready' | 'failed' | 'missing';
+        elapsedMs: number;
+        model?: string;
+        version?: string;
+        servers: Array<{
+            name: string;
+            status: 'connected' | 'pending' | 'failed' | 'needs-auth' | 'disabled' | 'unknown';
+            tools: string[];
+        }>;
+    };
+}
+
 /**
  * Validator Types
  *
@@ -433,6 +449,14 @@ declare function isStdioConfig(config: MCPConfig): config is StdioMCPConfig;
  */
 declare function isHttpConfig(config: MCPConfig): config is HttpMCPConfig;
 
+/** Native Cowork configuration only; this does not authorize the eval runner to
+ * execute writes. Unset/false preserves Cowork's existing approval behavior.
+ */
+declare const CoworkSetupConfigSchema: z.ZodObject<{
+    approveWriteTools: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>;
+type CoworkSetupConfig = z.infer<typeof CoworkSetupConfigSchema>;
+
 /** A tagged configuration block resolved by a public registry. */
 interface TaggedConfig {
     type: string;
@@ -461,6 +485,7 @@ interface EvalArm {
     scenarioTemplate?: string;
     metrics?: ExtensionConfig[];
     judges?: ExtensionConfig[];
+    coworkSetup?: CoworkSetupConfig;
 }
 /** A complete, organization-neutral evaluation manifest. */
 interface EvalManifest {
@@ -474,6 +499,7 @@ interface EvalManifest {
     arms?: EvalArm[];
     metrics?: ExtensionConfig[];
     judges?: ExtensionConfig[];
+    coworkSetup?: CoworkSetupConfig;
     results?: {
         store: ExtensionConfig;
     };
@@ -651,6 +677,9 @@ declare const EvalManifestSchema: z.ZodObject<{
         judges: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             type: z.ZodString;
         }, z.core.$loose>]>>>;
+        coworkSetup: z.ZodOptional<z.ZodObject<{
+            approveWriteTools: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>>;
     }, z.core.$strict>>>;
     metrics: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         type: z.ZodString;
@@ -658,6 +687,9 @@ declare const EvalManifestSchema: z.ZodObject<{
     judges: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         type: z.ZodString;
     }, z.core.$loose>]>>>;
+    coworkSetup: z.ZodOptional<z.ZodObject<{
+        approveWriteTools: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>;
     results: z.ZodOptional<z.ZodObject<{
         store: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             type: z.ZodString;
@@ -864,6 +896,8 @@ type CLIOutputFormat = 'stream-json' | 'json';
  * ```
  */
 interface CLIConfig {
+    /** Claude Code only: validate these MCP servers in its startup stream. */
+    claudeMcpServers?: string[];
     /** Child-process-only environment overrides. Undefined removes an inherited key. */
     env?: Record<string, string | undefined>;
     /**
@@ -1012,11 +1046,15 @@ interface LLMToolCall {
     id?: string;
     /** Tool result text, when the host surfaces it (paired to this call) */
     output?: string;
+    /** Explicit tool-result error status; absent when not observed. */
+    isError?: boolean;
 }
 /**
  * Result from an MCP host simulation
  */
 interface MCPHostSimulationResult {
+    /** Sanitized startup evidence, retained even when execution fails. */
+    diagnostics?: HostDiagnostics;
     /** Whether the simulation succeeded */
     success: boolean;
     /** Tool calls made by the LLM */
@@ -1752,6 +1790,7 @@ declare const EvalCaseSchema: z.ZodObject<{
                 "stream-json": "stream-json";
                 json: "json";
             }>>;
+            claudeMcpServers: z.ZodOptional<z.ZodArray<z.ZodString>>;
             timeout: z.ZodOptional<z.ZodNumber>;
         }, z.core.$strip>>;
         mcpServers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
@@ -2068,6 +2107,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
                     "stream-json": "stream-json";
                     json: "json";
                 }>>;
+                claudeMcpServers: z.ZodOptional<z.ZodArray<z.ZodString>>;
                 timeout: z.ZodOptional<z.ZodNumber>;
             }, z.core.$strip>>;
             mcpServers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
@@ -2626,10 +2666,14 @@ interface IterationResult {
             name: string;
         }>;
     };
+    /** Sanitized host evidence for this specific attempt. */
+    hostDiagnostics?: HostDiagnostics;
     /** Evidence level retained even when raw responses are redacted. */
     hostEvidence?: HostEvidence;
     /** Token usage from mcp_host LLM simulation in this iteration */
     hostUsage?: UsageMetrics;
+    /** Native numeric host measurements, retained after response redaction. */
+    hostTelemetry?: Record<string, unknown>;
     /** External host metadata for this iteration */
     externalHost?: ExternalHostMetadata;
 }
@@ -2809,6 +2853,8 @@ interface EvalCaseResult {
             name: string;
         }>;
     };
+    /** Sanitized host evidence; each iteration retains its own diagnostics. */
+    hostDiagnostics?: HostDiagnostics;
     /** Evidence level retained in persisted comparisons after response redaction. */
     hostEvidence?: HostEvidence;
     /**
@@ -2816,6 +2862,8 @@ interface EvalCaseResult {
      * Summed across all iterations. Only populated for mcp_host mode cases.
      */
     hostUsage?: UsageMetrics;
+    /** Native single-iteration measurements; multi-iteration values live in iterationResults. */
+    hostTelemetry?: Record<string, unknown>;
     /**
      * External host trace and evidence metadata.
      * Populated for external_host mode cases.
@@ -2943,6 +2991,9 @@ interface EvalExecutionResult {
     response: unknown;
     error?: string;
     hostUsage?: UsageMetrics;
+    hostTelemetry?: Record<string, unknown>;
+    /** Host time already spent outside the current evaluation timer. */
+    preExecutionDurationMs?: number;
     evidence?: HostEvidence;
 }
 
@@ -3306,14 +3357,29 @@ interface HostEvent {
     server?: string;
     arguments?: Record<string, unknown>;
     output?: string;
+    /** Explicit tool-result error status, absent when not observed. */
+    isError?: boolean;
+    rawName?: string;
     id?: string;
 }
 /** One execution trace. Hosts never return evaluation verdicts. */
 interface HostRunResult {
+    diagnostics?: HostDiagnostics;
     finalText: string;
     events: HostEvent[];
     error?: string;
     usage?: UsageMetrics;
+    /** Native and driver observations remain separately scoped; UI actions are not LLM usage. */
+    telemetry?: Record<string, unknown>;
+    /** Per-request wall time, excluding shared batch setup and cleanup. */
+    durationMs?: number;
+    llmDurationMs?: number;
+}
+interface HostBatchRequest {
+    caseId: string;
+    iteration: number;
+    input: HostRunInput;
+    config: HostConfig;
 }
 /** Public host extension point. */
 interface HostDefinition {
@@ -3322,6 +3388,8 @@ interface HostDefinition {
     createConfig?(options?: Record<string, unknown>): MCPHostConfig;
     /** Missing evidence declarations are treated as unverified. */
     readonly evidence?: HostEvidence;
+    /** Ordered traces for all selected iterations. The framework owns verdicts. */
+    runBatch?(requests: HostBatchRequest[], context: HostRunContext): Promise<HostRunResult[]>;
     run?(input: HostRunInput, config: HostConfig, context: HostRunContext): Promise<HostRunResult>;
 }
 /** Values emitted by a metric for one evaluation case. */
@@ -5310,6 +5378,7 @@ declare function runConformanceChecks(mcp: MCPFixtureApi, options?: MCPConforman
  * - 'none': No authentication
  */
 type AuthType = 'oauth' | 'api-token' | 'none';
+
 /**
  * Source of test results
  *
@@ -5370,4 +5439,4 @@ type ExpectationResultMap = Partial<Record<ExpectationType, EvalExpectationResul
  */
 type ExpectationBreakdown = Partial<Record<ExpectationType, number>>;
 
-export { type CompareEvalRunsOptions as $, type AuthType as A, type HostDriverConfig as B, type ExternalHostCapabilitiesConfig as C, type DatasetSource as D, type ExtensionConfig as E, type JudgeConfig as F, type Judge as G, type HostDefinition as H, BUILT_IN_RUBRICS as I, type JudgeMatcherOptions as J, type BuiltInRubric as K, type LLMProvider as L, type MCPFixtureApi as M, type BuiltInSanitizer as N, type OAuthSetupConfig as O, type PatternValidatorOptions as P, type CLIConfig as Q, type RubricSpec as R, type SchemaValidatorOptions as S, type TextValidatorOptions as T, CLIOAuthClient as U, type ValidationResult as V, type CLIOAuthClientConfig as W, type CLIOAuthResult as X, type CLIOutputFormat as Y, type CaseComparisonResult as Z, type ClientCredentialsConfig as _, type MCPHostConfig as a, type JudgeExpectConfig as a$, type ComparisonOutcome as a0, type ContentBlock as a1, type CreateMCPClientOptions as a2, type CustomJudgeExecutor as a3, type CustomJudgeResult as a4, type DatasetSourceContext as a5, DiscoveryError as a6, ENV_VAR_NAMES as a7, type EvalArm as a8, type EvalCase as a9, type ExpectationType as aA, type ExperimentMetric as aB, type ExternalHostCapabilityBinding as aC, type ExternalHostCapabilityContext as aD, type ExternalHostFailureKind as aE, type ExternalHostMetadata as aF, type ExternalHostSession as aG, type ExternalHostSimulationResult as aH, type ExternalHostType as aI, type FieldRemovalSanitizer as aJ, FileEvalResultStore as aK, type FileEvalResultStoreConfig as aL, GCSEvalResultStore as aM, type GCSEvalResultStoreConfig as aN, type HostArtifact as aO, type HostCapability as aP, type HostConfig as aQ, type HostConfigPatch as aR, type HostEvent as aS, type HostEvidence as aT, type HostRunContext$1 as aU, type HostRunInput as aV, type HostRunOptions as aW, type HostRunResult as aX, type HostType as aY, type HttpMCPConfig as aZ, type IterationResult as a_, type EvalCaseComparison as aa, type EvalCaseComparisonOutcome as ab, type EvalCaseRequest as ac, EvalCaseSchema as ad, type EvalContext as ae, EvalDatasetSchema as af, type EvalExpectBlock as ag, type EvalExpectationResult as ah, type EvalManifestInput as ai, EvalManifestSchema as aj, type EvalMode as ak, type EvalResultStore as al, type EvalResultStoreConfig as am, type EvalResultStoreLike as an, type EvalRunComparisonLabels as ao, type EvalRunComparisonResult as ap, type EvalRunMetadata as aq, type EvalRunnerOptions as ar, type EvalSummaryGenerator as as, type EvaluationArmResult as at, type HostRunContext as au, type EvaluationSuiteOptions as av, type EvaluationSuiteResult as aw, type EvidenceSource as ax, type ExpectationBreakdown as ay, type ExpectationResultMap as az, type MCPHostSimulationResult as b, type TraceSource as b$, type JudgeResult as b0, type JudgeValidatorConfig as b1, type LLMToolCall as b2, type ListStoredArtifactsOptions as b3, type LoadDatasetOptions as b4, type MCPAuthConfig as b5, type MCPAuthFixtures as b6, type MCPClientCredentialsConfig as b7, MCPConfigSchema as b8, type MCPConformanceCheck as b9, type RunSummary as bA, type RunTelemetry as bB, type SaveBaselineOptions as bC, type SaveEvalRunComparisonOptions as bD, type SaveServerComparisonOptions as bE, type SchemaRegistry as bF, type SerializedEvalDataset as bG, type ServerComparisonOptions as bH, type ServerComparisonResult as bI, SnapshotSanitizers as bJ, type StdioMCPConfig as bK, type StoredArtifactKind as bL, type StoredArtifactSummary as bM, type StoredClientInfo as bN, type StoredEvalArtifact as bO, type StoredEvalArtifactMetadata as bP, type StoredEvalResultLoadOptions as bQ, type StoredEvalResultRef as bR, type StoredEvalResultSaveOptions as bS, type StoredEvalRunRef as bT, type StoredOAuthState as bU, type StoredServerMetadata as bV, type StoredTokens as bW, type TaggedConfig as bX, type TokenResult as bY, type ToolMetadataOverride as bZ, type ToolOverrideVariant as b_, type MCPConformanceOptions as ba, type MCPConformanceRaw as bb, type MCPConformanceResult as bc, type MCPConformanceResultData as bd, type MCPEvalData as be, type MCPEvalHistoricalSummary as bf, type MCPEvalReporterConfig as bg, type MCPEvalRunData as bh, type MCPFixtureOptions as bi, type MCPHostCapabilities as bj, type MCPHostSimulator as bk, type MCPOAuthConfig as bl, type MCPServerCapabilitiesData as bm, MCP_PROTOCOL_VERSION as bn, type MetricKind as bo, type NormalizedToolResponse as bp, type ObservationConfidence as bq, PlaywrightOAuthClientProvider as br, type PlaywrightOAuthClientProviderConfig as bs, type PredicateResult as bt, type ProposeVariantsContext as bu, type ProtectedResourceDiscoveryResult as bv, type ProtectedResourceMetadata as bw, type ProviderKind as bx, type RegexSanitizer as by, type ResultSource as bz, type SizeValidatorOptions as c, type UsageMetrics as c0, type VariantCandidateResult as c1, type VariantExperimentOptions as c2, type VariantExperimentReason as c3, type VariantExperimentResult as c4, type VariantExperimentRound as c5, type VariantImprovementProposal as c6, type VariantRecommendation as c7, closeMCPClient as c8, compareEvalRuns as c9, refreshAccessToken as cA, registerJudge as cB, resolveDatasetPaths as cC, resolveEvalResultStore as cD, resolveRubric as cE, runConformanceChecks as cF, runEvalCase as cG, runEvalDataset as cH, runServerComparison as cI, runVariantExperiment as cJ, saveBaseline as cK, saveEvalRunComparison as cL, saveServerComparison as cM, validateEvalCase as cN, validateEvalDataset as cO, validateJudge as cP, validateMCPConfig as cQ, validateToolCallCount as cR, validateToolCalls as cS, createDefaultArtifactId as ca, createEvalResultStore as cb, createMCPClientForConfig as cc, createMCPFixture as cd, createStoredEvalArtifact as ce, defaultEnvironmentMetadata as cf, discoverAuthorizationServer as cg, discoverProtectedResource as ch, extractText as ci, hasValidTokens as cj, injectTokens as ck, isBuiltInRubric as cl, isEvalResultStore as cm, isHttpConfig as cn, isStdioConfig as co, loadBaseline as cp, loadEvalDataset as cq, loadEvalDatasetFromObject as cr, loadEvalManifest as cs, loadEvalManifestFromObject as ct, loadStoredEvalRunnerResult as cu, loadTokens as cv, loadTokensFromEnv as cw, test as cx, normalizeToolResponse as cy, performClientCredentialsFlow as cz, type SnapshotSanitizer as d, type ToolPredicate as e, type ToolCallExpectation as f, type ToolCallCountOptions as g, type JudgeDefinition as h, type MetricDefinition as i, type ResultStoreDefinition as j, type EvalManifest as k, type EvalCaseResult as l, type MetricValue as m, type ResolvedMetric as n, type EvalDataset as o, type MCPConfig as p, type DatasetConfig as q, type EvalRunnerResult as r, type EvaluationSummary as s, type EvaluationBatchItem as t, type EvaluationBatchOptions as u, type EvaluationBatchResult as v, type ExternalHostConfig as w, type ExternalHostRunResult as x, type ExternalHostCapabilityImplementation as y, type HostDriverId as z };
+export { type CompareEvalRunsOptions as $, type AuthType as A, type HostDriverConfig as B, type ExternalHostCapabilitiesConfig as C, type DatasetSource as D, type ExtensionConfig as E, type JudgeConfig as F, type Judge as G, type HostDefinition as H, BUILT_IN_RUBRICS as I, type JudgeMatcherOptions as J, type BuiltInRubric as K, type LLMProvider as L, type MCPFixtureApi as M, type BuiltInSanitizer as N, type OAuthSetupConfig as O, type PatternValidatorOptions as P, type CLIConfig as Q, type RubricSpec as R, type SchemaValidatorOptions as S, type TextValidatorOptions as T, CLIOAuthClient as U, type ValidationResult as V, type CLIOAuthClientConfig as W, type CLIOAuthResult as X, type CLIOutputFormat as Y, type CaseComparisonResult as Z, type ClientCredentialsConfig as _, type MCPHostConfig as a, type HttpMCPConfig as a$, type ComparisonOutcome as a0, type ContentBlock as a1, type CreateMCPClientOptions as a2, type CustomJudgeExecutor as a3, type CustomJudgeResult as a4, type DatasetSourceContext as a5, DiscoveryError as a6, ENV_VAR_NAMES as a7, type EvalArm as a8, type EvalCase as a9, type ExpectationType as aA, type ExperimentMetric as aB, type ExternalHostCapabilityBinding as aC, type ExternalHostCapabilityContext as aD, type ExternalHostFailureKind as aE, type ExternalHostMetadata as aF, type ExternalHostSession as aG, type ExternalHostSimulationResult as aH, type ExternalHostType as aI, type FieldRemovalSanitizer as aJ, FileEvalResultStore as aK, type FileEvalResultStoreConfig as aL, GCSEvalResultStore as aM, type GCSEvalResultStoreConfig as aN, type HostArtifact as aO, type HostBatchRequest as aP, type HostCapability as aQ, type HostConfig as aR, type HostConfigPatch as aS, type HostDiagnostics as aT, type HostEvent as aU, type HostEvidence as aV, type HostRunContext$1 as aW, type HostRunInput as aX, type HostRunOptions as aY, type HostRunResult as aZ, type HostType as a_, type EvalCaseComparison as aa, type EvalCaseComparisonOutcome as ab, type EvalCaseRequest as ac, EvalCaseSchema as ad, type EvalContext as ae, EvalDatasetSchema as af, type EvalExpectBlock as ag, type EvalExpectationResult as ah, type EvalManifestInput as ai, EvalManifestSchema as aj, type EvalMode as ak, type EvalResultStore as al, type EvalResultStoreConfig as am, type EvalResultStoreLike as an, type EvalRunComparisonLabels as ao, type EvalRunComparisonResult as ap, type EvalRunMetadata as aq, type EvalRunnerOptions as ar, type EvalSummaryGenerator as as, type EvaluationArmResult as at, type HostRunContext as au, type EvaluationSuiteOptions as av, type EvaluationSuiteResult as aw, type EvidenceSource as ax, type ExpectationBreakdown as ay, type ExpectationResultMap as az, type MCPHostSimulationResult as b, type ToolMetadataOverride as b$, type IterationResult as b0, type JudgeExpectConfig as b1, type JudgeResult as b2, type JudgeValidatorConfig as b3, type LLMToolCall as b4, type ListStoredArtifactsOptions as b5, type LoadDatasetOptions as b6, type MCPAuthConfig as b7, type MCPAuthFixtures as b8, type MCPClientCredentialsConfig as b9, type RegexSanitizer as bA, type ResultSource as bB, type RunSummary as bC, type RunTelemetry as bD, type SaveBaselineOptions as bE, type SaveEvalRunComparisonOptions as bF, type SaveServerComparisonOptions as bG, type SchemaRegistry as bH, type SerializedEvalDataset as bI, type ServerComparisonOptions as bJ, type ServerComparisonResult as bK, SnapshotSanitizers as bL, type StdioMCPConfig as bM, type StoredArtifactKind as bN, type StoredArtifactSummary as bO, type StoredClientInfo as bP, type StoredEvalArtifact as bQ, type StoredEvalArtifactMetadata as bR, type StoredEvalResultLoadOptions as bS, type StoredEvalResultRef as bT, type StoredEvalResultSaveOptions as bU, type StoredEvalRunRef as bV, type StoredOAuthState as bW, type StoredServerMetadata as bX, type StoredTokens as bY, type TaggedConfig as bZ, type TokenResult as b_, MCPConfigSchema as ba, type MCPConformanceCheck as bb, type MCPConformanceOptions as bc, type MCPConformanceRaw as bd, type MCPConformanceResult as be, type MCPConformanceResultData as bf, type MCPEvalData as bg, type MCPEvalHistoricalSummary as bh, type MCPEvalReporterConfig as bi, type MCPEvalRunData as bj, type MCPFixtureOptions as bk, type MCPHostCapabilities as bl, type MCPHostSimulator as bm, type MCPOAuthConfig as bn, type MCPServerCapabilitiesData as bo, MCP_PROTOCOL_VERSION as bp, type MetricKind as bq, type NormalizedToolResponse as br, type ObservationConfidence as bs, PlaywrightOAuthClientProvider as bt, type PlaywrightOAuthClientProviderConfig as bu, type PredicateResult as bv, type ProposeVariantsContext as bw, type ProtectedResourceDiscoveryResult as bx, type ProtectedResourceMetadata as by, type ProviderKind as bz, type SizeValidatorOptions as c, type ToolOverrideVariant as c0, type TraceSource as c1, type UsageMetrics as c2, type VariantCandidateResult as c3, type VariantExperimentOptions as c4, type VariantExperimentReason as c5, type VariantExperimentResult as c6, type VariantExperimentRound as c7, type VariantImprovementProposal as c8, type VariantRecommendation as c9, normalizeToolResponse as cA, performClientCredentialsFlow as cB, refreshAccessToken as cC, registerJudge as cD, resolveDatasetPaths as cE, resolveEvalResultStore as cF, resolveRubric as cG, runConformanceChecks as cH, runEvalCase as cI, runEvalDataset as cJ, runServerComparison as cK, runVariantExperiment as cL, saveBaseline as cM, saveEvalRunComparison as cN, saveServerComparison as cO, validateEvalCase as cP, validateEvalDataset as cQ, validateJudge as cR, validateMCPConfig as cS, validateToolCallCount as cT, validateToolCalls as cU, closeMCPClient as ca, compareEvalRuns as cb, createDefaultArtifactId as cc, createEvalResultStore as cd, createMCPClientForConfig as ce, createMCPFixture as cf, createStoredEvalArtifact as cg, defaultEnvironmentMetadata as ch, discoverAuthorizationServer as ci, discoverProtectedResource as cj, extractText as ck, hasValidTokens as cl, injectTokens as cm, isBuiltInRubric as cn, isEvalResultStore as co, isHttpConfig as cp, isStdioConfig as cq, loadBaseline as cr, loadEvalDataset as cs, loadEvalDatasetFromObject as ct, loadEvalManifest as cu, loadEvalManifestFromObject as cv, loadStoredEvalRunnerResult as cw, loadTokens as cx, loadTokensFromEnv as cy, test as cz, type SnapshotSanitizer as d, type ToolPredicate as e, type ToolCallExpectation as f, type ToolCallCountOptions as g, type JudgeDefinition as h, type MetricDefinition as i, type ResultStoreDefinition as j, type EvalManifest as k, type EvalCaseResult as l, type MetricValue as m, type ResolvedMetric as n, type EvalDataset as o, type MCPConfig as p, type DatasetConfig as q, type EvalRunnerResult as r, type EvaluationSummary as s, type EvaluationBatchItem as t, type EvaluationBatchOptions as u, type EvaluationBatchResult as v, type ExternalHostConfig as w, type ExternalHostRunResult as x, type ExternalHostCapabilityImplementation as y, type HostDriverId as z };