@gleanwork/mcp-server-tester 2.0.0-beta.3 → 2.0.0-beta.5

package/dist/{index-CwFHKIoN.d.cts → index-BTEdKnrn.d.cts}

@@ -1038,6 +1038,9 @@ interface LLMToolCall {
     server?: string;
     /** Original provider-encoded name, retained for diagnostics. */
     rawName?: string;
+    durationMs?: number;
+    startedAt?: string;
+    completedAt?: string;
     /** Tool name */
     name: string;
     /** Tool arguments (as provided by LLM) */
@@ -1114,11 +1117,77 @@ interface MCPHostSimulator {
     signal?: AbortSignal): Promise<MCPHostSimulationResult>;
 }
 
+declare const CodexSetupConfigSchema: z.ZodObject<{
+    configPath: z.ZodOptional<z.ZodString>;
+    configName: z.ZodOptional<z.ZodString>;
+    servers: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        transport: z.ZodLiteral<"stdio">;
+        label: z.ZodString;
+        command: z.ZodString;
+        args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        cwd: z.ZodOptional<z.ZodString>;
+        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    }, z.core.$strict>, z.ZodObject<{
+        transport: z.ZodLiteral<"http">;
+        label: z.ZodString;
+        url: z.ZodString;
+        bearerTokenEnvVar: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>], "transport">>>;
+    configs: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        servers: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            transport: z.ZodLiteral<"stdio">;
+            label: z.ZodString;
+            command: z.ZodString;
+            args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            cwd: z.ZodOptional<z.ZodString>;
+            env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        }, z.core.$strict>, z.ZodObject<{
+            transport: z.ZodLiteral<"http">;
+            label: z.ZodString;
+            url: z.ZodString;
+            bearerTokenEnvVar: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>], "transport">>;
+    }, z.core.$strict>>>;
+}, z.core.$strict>;
+type CodexSetupConfig = z.infer<typeof CodexSetupConfigSchema>;
+
+declare const COMPUTER_USE_TOKEN_FIELDS: readonly ["input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"];
+type ComputerUseTokenField = (typeof COMPUTER_USE_TOKEN_FIELDS)[number];
+/** Usage totals cover only observed fields on completed planner responses. */
+interface ComputerUseTelemetry {
+    accounting: 'complete' | 'partial';
+    response_models: string[];
+    planner_response_count: number;
+    usage: Partial<Record<ComputerUseTokenField, number>>;
+    usage_observation_counts: Record<ComputerUseTokenField, number>;
+    duration_ms: number;
+    action_count: number;
+    attempted_action_count: number;
+    executed_action_count: number;
+    refused_action_count: number;
+    cost: {
+        status: 'unavailable';
+    };
+}
+interface SemanticDesktopTelemetry {
+    driver: 'linux-desktop';
+    accounting: 'complete' | 'partial';
+    duration_ms: number;
+    action_count: number;
+    planner: {
+        status: 'not-applicable';
+    };
+    cost: {
+        status: 'not-applicable';
+    };
+}
+
 type ExternalHostType = 'cli' | 'browser' | 'desktop' | 'custom';
 type HostCapability = 'control' | 'input' | 'completion' | 'trace' | 'normalize';
 type TraceSource = 'mcp-proxy' | 'mcp-server-logs' | 'host-local-transcript' | 'host-native-export' | 'browser-api' | 'accessibility' | 'dom' | 'screenshot' | 'stdout' | 'manual-import' | 'none';
 type ObservationConfidence = 'high' | 'medium' | 'low' | 'unknown';
-type ExternalHostCorrelationStrategy = 'prompt_marker' | 'host_session_metadata' | 'none';
+type ExternalHostCorrelationStrategy = 'exact_prompt' | 'prompt_marker' | 'host_session_metadata' | 'none';
 interface HostDriverId {
     provider: string;
     product: string;
@@ -1128,7 +1197,7 @@ interface HostDriverId {
     channel?: string;
 }
 type HostDriverConfig = HostDriverId | string;
-type ExternalHostFailureKind = 'app_unavailable' | 'automation_permission_denied' | 'submission_failed' | 'no_matching_session' | 'ambiguous_matching_sessions' | 'timeout' | 'parse_failure' | 'host_run_failed' | 'unsupported_host' | 'unknown';
+type ExternalHostFailureKind = 'app_unavailable' | 'automation_permission_denied' | 'submission_failed' | 'no_matching_session' | 'ambiguous_matching_sessions' | 'timeout' | 'parse_failure' | 'host_run_failed' | 'cleanup_failed' | 'unsupported_host' | 'unknown';
 interface HostArtifact {
     kind: 'stdout' | 'stderr' | 'log' | 'transcript' | 'audit' | 'metadata' | 'screenshot' | 'video' | 'har' | 'trace';
     name: string;
@@ -1136,10 +1205,52 @@ interface HostArtifact {
     contentType?: string;
     summary?: string;
 }
+interface ExternalHostTelemetry {
+    resultCount?: number;
+    apiCallCount?: number;
+    models?: string[];
+    inputTokens?: number;
+    outputTokens?: number;
+    cacheReadInputTokens?: number;
+    cacheCreationInputTokens?: number;
+    totalCostUsd?: number;
+    durationMs?: number;
+    durationApiMs?: number;
+    toolCallCount?: number;
+    toolErrorCount?: number;
+    mcpToolCallCount?: number;
+    mcpToolErrorCount?: number;
+    hostToolCallCount?: number;
+    hostToolErrorCount?: number;
+    hostToolDurationMs?: number;
+    /** Wall-clock union of all native tool intervals, including built-in host tools. */
+    toolWallDurationMs?: number;
+    /** Original native identities, independent of framework event source normalization. */
+    toolProvenance?: Array<{
+        id?: string;
+        source: 'mcp' | 'host';
+        nativeServer?: string;
+        nativeTool: string;
+        nativeItemType?: string;
+        /** Native call has no recorded output yet. */
+        pending?: boolean;
+        /** Referenced inside a code-mode `exec` runner; no own arguments or timing. */
+        viaCodeMode?: boolean;
+    }>;
+    /** The turn did not complete; calls and usage cover only what was recorded. */
+    partial?: boolean;
+    reasoningOutputTokens?: number;
+    reasoningEffort?: string;
+    timeToFirstTokenMs?: number;
+    /** Wall-clock union of MCP call intervals; excludes overlap. */
+    mcpWallDurationMs?: number;
+}
 interface ExternalHostSession {
     id?: string;
-    runMarker: string;
+    /** Absent when no marker was sent to the native host. */
+    runMarker?: string;
     requestId?: string;
+    turnId?: string;
     cliSessionId?: string;
     cwd?: string;
     startedAt?: string;
@@ -1149,6 +1260,7 @@ interface ExternalHostCorrelationConfig {
     /**
      * How this run should be correlated with host-native evidence.
      *
+     * - exact_prompt: match unchanged user text in a fresh native session.
      * - prompt_marker: append a marker to the submitted prompt.
      * - host_session_metadata: rely on host-native session metadata.
      * - none: no host-visible marker is submitted.
@@ -1166,8 +1278,15 @@ interface ExternalHostCorrelationConfig {
 }
 interface ExternalHostCorrelationMetadata {
     strategy: ExternalHostCorrelationStrategy;
+    /** Internal run identifier; not sent to the host unless includedInPrompt is true. */
     marker: string;
     includedInPrompt: boolean;
+    /** SHA-256 of the exact submitted UTF-8 prompt, where supported. */
+    promptSha256?: string;
+    promptUnchanged?: boolean;
+    /** Native user text: literal match, or the app's single appended LF. */
+    nativePromptMatch?: 'exact' | 'native_terminal_lf' | 'native_markdown_escaped';
+    nativePromptSha256?: string;
 }
 interface ExternalHostMetadata {
     driver: HostDriverId;
@@ -1190,6 +1309,26 @@ interface ExternalHostMetadata {
         usage?: TraceSource;
         cost?: TraceSource;
     };
+    telemetry?: ExternalHostTelemetry;
+    /** Controller API accounting is separate from the evaluated application's usage. */
+    computerUse?: {
+        provider: 'anthropic-computer-use';
+        submission: {
+            status: 'completed' | 'failed';
+            telemetry?: ComputerUseTelemetry;
+        };
+    };
+    /** Deterministic native UI accounting; never reported as planner/model usage. */
+    nativeController?: {
+        provider: 'linux-atspi';
+        surface: 'chatgpt-work' | 'codex';
+        submission: {
+            status: 'completed' | 'failed';
+            telemetry?: SemanticDesktopTelemetry;
+            /** Failure-only sanitized UI state: surface, counts, and text hash. Never text. */
+            draftState?: Record<string, string | number | boolean>;
+        };
+    };
     evidence?: {
         finalAnswer?: EvidenceSource;
         toolCalls?: EvidenceSource;
@@ -1219,6 +1358,9 @@ interface ExternalHostConfig {
      * End-to-end timeout for the host run.
      */
     timeoutMs?: number;
+    /** Requested ChatGPT model ID (verified against the native turn record). */
+    model?: string;
+    reasoningEffort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max' | 'ultra';
     /**
      * Capability bindings used to compose this external host runner.
      * If omitted, the runtime may provide a built-in default for known drivers.
@@ -1228,6 +1370,10 @@ interface ExternalHostConfig {
      * Run correlation strategy. Built-in drivers may provide defaults.
      */
     correlation?: ExternalHostCorrelationConfig;
+    /**
+     * Optional managed MCP configuration lifecycle for the ChatGPT desktop driver.
+     */
+    codexSetup?: CodexSetupConfig;
     /**
      * Driver-wide options available to capability implementations.
      */
@@ -1260,6 +1406,9 @@ interface ExternalHostRunFailure {
     success: false;
     error: string;
     toolCalls: LLMToolCall[];
+    /** Partial native evidence from a bound turn that did not complete. */
+    conversationHistory?: MCPHostSimulationResult['conversationHistory'];
+    usage?: UsageMetrics;
     externalHost: ExternalHostMetadata;
 }
 type ExternalHostRunResult = ExternalHostRunSuccess | ExternalHostRunFailure;
@@ -1299,6 +1448,7 @@ interface ExternalHostCapabilityImplementation {
     capabilities: HostCapability[];
     setup?(context: ExternalHostCapabilityContext): Promise<ExternalHostRunResult | void>;
     run?(context: ExternalHostCapabilityContext): Promise<ExternalHostRunResult | void>;
+    teardown?(context: ExternalHostCapabilityContext): Promise<void>;
 }
 interface EvidenceSource {
     source: TraceSource;
@@ -1343,7 +1493,9 @@ interface UsageMetrics {
     /**
      * Total cost in USD
      */
-    totalCostUsd: number;
+    totalCostUsd?: number;
+    /** Reasoning tokens, already included in outputTokens. */
+    reasoningOutputTokens?: number;
     /**
      * Execution duration in milliseconds
      */
@@ -1835,6 +1987,15 @@ declare const EvalCaseSchema: z.ZodObject<{
         }>>;
         variant: z.ZodOptional<z.ZodString>;
         timeoutMs: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        reasoningEffort: z.ZodOptional<z.ZodEnum<{
+            low: "low";
+            medium: "medium";
+            high: "high";
+            xhigh: "xhigh";
+            max: "max";
+            ultra: "ultra";
+        }>>;
         capabilities: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
             input: "input";
             normalize: "normalize";
@@ -1865,12 +2026,46 @@ declare const EvalCaseSchema: z.ZodObject<{
         correlation: z.ZodOptional<z.ZodObject<{
             strategy: z.ZodOptional<z.ZodEnum<{
                 none: "none";
+                exact_prompt: "exact_prompt";
                 prompt_marker: "prompt_marker";
                 host_session_metadata: "host_session_metadata";
             }>>;
             includeInPrompt: z.ZodOptional<z.ZodBoolean>;
             promptTemplate: z.ZodOptional<z.ZodString>;
         }, z.core.$strip>>;
+        codexSetup: z.ZodOptional<z.ZodObject<{
+            configPath: z.ZodOptional<z.ZodString>;
+            configName: z.ZodOptional<z.ZodString>;
+            servers: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                transport: z.ZodLiteral<"stdio">;
+                label: z.ZodString;
+                command: z.ZodString;
+                args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                cwd: z.ZodOptional<z.ZodString>;
+                env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            }, z.core.$strict>, z.ZodObject<{
+                transport: z.ZodLiteral<"http">;
+                label: z.ZodString;
+                url: z.ZodString;
+                bearerTokenEnvVar: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>], "transport">>>;
+            configs: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                servers: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    transport: z.ZodLiteral<"stdio">;
+                    label: z.ZodString;
+                    command: z.ZodString;
+                    args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                    cwd: z.ZodOptional<z.ZodString>;
+                    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+                }, z.core.$strict>, z.ZodObject<{
+                    transport: z.ZodLiteral<"http">;
+                    label: z.ZodString;
+                    url: z.ZodString;
+                    bearerTokenEnvVar: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>], "transport">>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
         options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     }, z.core.$strip>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
@@ -2152,6 +2347,15 @@ declare const EvalDatasetSchema: z.ZodObject<{
             }>>;
             variant: z.ZodOptional<z.ZodString>;
             timeoutMs: z.ZodOptional<z.ZodNumber>;
+            model: z.ZodOptional<z.ZodString>;
+            reasoningEffort: z.ZodOptional<z.ZodEnum<{
+                low: "low";
+                medium: "medium";
+                high: "high";
+                xhigh: "xhigh";
+                max: "max";
+                ultra: "ultra";
+            }>>;
             capabilities: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
                 input: "input";
                 normalize: "normalize";
@@ -2182,12 +2386,46 @@ declare const EvalDatasetSchema: z.ZodObject<{
             correlation: z.ZodOptional<z.ZodObject<{
                 strategy: z.ZodOptional<z.ZodEnum<{
                     none: "none";
+                    exact_prompt: "exact_prompt";
                     prompt_marker: "prompt_marker";
                     host_session_metadata: "host_session_metadata";
                 }>>;
                 includeInPrompt: z.ZodOptional<z.ZodBoolean>;
                 promptTemplate: z.ZodOptional<z.ZodString>;
             }, z.core.$strip>>;
+            codexSetup: z.ZodOptional<z.ZodObject<{
+                configPath: z.ZodOptional<z.ZodString>;
+                configName: z.ZodOptional<z.ZodString>;
+                servers: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    transport: z.ZodLiteral<"stdio">;
+                    label: z.ZodString;
+                    command: z.ZodString;
+                    args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                    cwd: z.ZodOptional<z.ZodString>;
+                    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+                }, z.core.$strict>, z.ZodObject<{
+                    transport: z.ZodLiteral<"http">;
+                    label: z.ZodString;
+                    url: z.ZodString;
+                    bearerTokenEnvVar: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>], "transport">>>;
+                configs: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodString;
+                    servers: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        transport: z.ZodLiteral<"stdio">;
+                        label: z.ZodString;
+                        command: z.ZodString;
+                        args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                        cwd: z.ZodOptional<z.ZodString>;
+                        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+                    }, z.core.$strict>, z.ZodObject<{
+                        transport: z.ZodLiteral<"http">;
+                        label: z.ZodString;
+                        url: z.ZodString;
+                        bearerTokenEnvVar: z.ZodOptional<z.ZodString>;
+                    }, z.core.$strict>], "transport">>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>>;
             options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
         }, z.core.$strip>>;
         metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
@@ -3361,6 +3599,9 @@ interface HostEvent {
     isError?: boolean;
     rawName?: string;
     id?: string;
+    durationMs?: number;
+    startedAt?: string;
+    completedAt?: string;
 }
 /** One execution trace. Hosts never return evaluation verdicts. */
 interface HostRunResult {