@wix/evalforge-types 0.3.0

package/build/types/evaluation/eval-run.d.ts

@@ -0,0 +1,581 @@
+import { z } from 'zod';
+/**
+ * Trigger types for evaluations.
+ */
+export declare enum TriggerType {
+    RESOURCES_UPDATED = "RESOURCES_UPDATED",
+    MCP_VERSION_RELEASE = "MCP_VERSION_RELEASE",
+    MCP_PREVIEW_CREATED = "MCP_PREVIEW_CREATED",
+    MANUAL = "MANUAL"
+}
+/**
+ * Trigger metadata schema.
+ */
+export declare const TriggerMetadataSchema: z.ZodObject<{
+    version: z.ZodOptional<z.ZodString>;
+    resourceUpdated: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export type TriggerMetadata = z.infer<typeof TriggerMetadataSchema>;
+/**
+ * Trigger schema.
+ */
+export declare const TriggerSchema: z.ZodObject<{
+    id: z.ZodString;
+    metadata: z.ZodOptional<z.ZodObject<{
+        version: z.ZodOptional<z.ZodString>;
+        resourceUpdated: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    type: z.ZodEnum<typeof TriggerType>;
+}, z.core.$strip>;
+export type Trigger = z.infer<typeof TriggerSchema>;
+/**
+ * Failure category enum.
+ */
+export declare enum FailureCategory {
+    MISSING_FILE = "missing_file",
+    WRONG_CONTENT = "wrong_content",
+    BUILD_ERROR = "build_error",
+    TEST_FAILURE = "test_failure",
+    RUNTIME_ERROR = "runtime_error",
+    PERFORMANCE = "performance"
+}
+/**
+ * Failure severity enum.
+ */
+export declare enum FailureSeverity {
+    CRITICAL = "critical",
+    HIGH = "high",
+    MEDIUM = "medium",
+    LOW = "low"
+}
+/**
+ * Diff line type schema.
+ */
+export declare const DiffLineTypeSchema: z.ZodEnum<{
+    added: "added";
+    removed: "removed";
+    unchanged: "unchanged";
+}>;
+export type DiffLineType = z.infer<typeof DiffLineTypeSchema>;
+/**
+ * Diff line schema - represents a single line in a diff.
+ */
+export declare const DiffLineSchema: z.ZodObject<{
+    type: z.ZodEnum<{
+        added: "added";
+        removed: "removed";
+        unchanged: "unchanged";
+    }>;
+    content: z.ZodString;
+    lineNumber: z.ZodNumber;
+}, z.core.$strip>;
+export type DiffLine = z.infer<typeof DiffLineSchema>;
+/**
+ * Diff content schema - represents a file diff.
+ */
+export declare const DiffContentSchema: z.ZodObject<{
+    path: z.ZodString;
+    expected: z.ZodString;
+    actual: z.ZodString;
+    diffLines: z.ZodArray<z.ZodObject<{
+        type: z.ZodEnum<{
+            added: "added";
+            removed: "removed";
+            unchanged: "unchanged";
+        }>;
+        content: z.ZodString;
+        lineNumber: z.ZodNumber;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type DiffContent = z.infer<typeof DiffContentSchema>;
+/**
+ * Command execution schema.
+ */
+export declare const CommandExecutionSchema: z.ZodObject<{
+    command: z.ZodString;
+    exitCode: z.ZodNumber;
+    output: z.ZodOptional<z.ZodString>;
+    duration: z.ZodNumber;
+}, z.core.$strip>;
+export type CommandExecution = z.infer<typeof CommandExecutionSchema>;
+/**
+ * File modification schema.
+ */
+export declare const FileModificationSchema: z.ZodObject<{
+    path: z.ZodString;
+    action: z.ZodEnum<{
+        deleted: "deleted";
+        created: "created";
+        modified: "modified";
+    }>;
+}, z.core.$strip>;
+export type FileModification = z.infer<typeof FileModificationSchema>;
+/**
+ * API call schema.
+ */
+export declare const ApiCallSchema: z.ZodObject<{
+    endpoint: z.ZodString;
+    tokensUsed: z.ZodNumber;
+    duration: z.ZodNumber;
+}, z.core.$strip>;
+export type ApiCall = z.infer<typeof ApiCallSchema>;
+/**
+ * Execution trace schema - represents detailed execution information.
+ */
+export declare const ExecutionTraceSchema: z.ZodObject<{
+    commands: z.ZodArray<z.ZodObject<{
+        command: z.ZodString;
+        exitCode: z.ZodNumber;
+        output: z.ZodOptional<z.ZodString>;
+        duration: z.ZodNumber;
+    }, z.core.$strip>>;
+    filesModified: z.ZodArray<z.ZodObject<{
+        path: z.ZodString;
+        action: z.ZodEnum<{
+            deleted: "deleted";
+            created: "created";
+            modified: "modified";
+        }>;
+    }, z.core.$strip>>;
+    apiCalls: z.ZodArray<z.ZodObject<{
+        endpoint: z.ZodString;
+        tokensUsed: z.ZodNumber;
+        duration: z.ZodNumber;
+    }, z.core.$strip>>;
+    totalDuration: z.ZodNumber;
+}, z.core.$strip>;
+export type ExecutionTrace = z.infer<typeof ExecutionTraceSchema>;
+/**
+ * Failure analysis schema.
+ */
+export declare const FailureAnalysisSchema: z.ZodObject<{
+    category: z.ZodEnum<typeof FailureCategory>;
+    severity: z.ZodEnum<typeof FailureSeverity>;
+    summary: z.ZodString;
+    details: z.ZodString;
+    rootCause: z.ZodString;
+    suggestedFix: z.ZodString;
+    relatedAssertions: z.ZodArray<z.ZodString>;
+    codeSnippet: z.ZodOptional<z.ZodString>;
+    similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    patternId: z.ZodOptional<z.ZodString>;
+    diff: z.ZodOptional<z.ZodObject<{
+        path: z.ZodString;
+        expected: z.ZodString;
+        actual: z.ZodString;
+        diffLines: z.ZodArray<z.ZodObject<{
+            type: z.ZodEnum<{
+                added: "added";
+                removed: "removed";
+                unchanged: "unchanged";
+            }>;
+            content: z.ZodString;
+            lineNumber: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    executionTrace: z.ZodOptional<z.ZodObject<{
+        commands: z.ZodArray<z.ZodObject<{
+            command: z.ZodString;
+            exitCode: z.ZodNumber;
+            output: z.ZodOptional<z.ZodString>;
+            duration: z.ZodNumber;
+        }, z.core.$strip>>;
+        filesModified: z.ZodArray<z.ZodObject<{
+            path: z.ZodString;
+            action: z.ZodEnum<{
+                deleted: "deleted";
+                created: "created";
+                modified: "modified";
+            }>;
+        }, z.core.$strip>>;
+        apiCalls: z.ZodArray<z.ZodObject<{
+            endpoint: z.ZodString;
+            tokensUsed: z.ZodNumber;
+            duration: z.ZodNumber;
+        }, z.core.$strip>>;
+        totalDuration: z.ZodNumber;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type FailureAnalysis = z.infer<typeof FailureAnalysisSchema>;
+/**
+ * Evaluation run schema.
+ *
+ * Represents a complete evaluation run with configuration, results, and metrics.
+ */
+export declare const EvalRunSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    createdAt: z.ZodString;
+    updatedAt: z.ZodString;
+    deleted: z.ZodOptional<z.ZodBoolean>;
+    projectId: z.ZodString;
+    agentId: z.ZodOptional<z.ZodString>;
+    skillsGroupId: z.ZodOptional<z.ZodString>;
+    scenarioIds: z.ZodArray<z.ZodString>;
+    status: z.ZodEnum<typeof import("./metrics.js").EvalStatus>;
+    progress: z.ZodNumber;
+    results: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        targetId: z.ZodString;
+        targetName: z.ZodOptional<z.ZodString>;
+        scenarioId: z.ZodString;
+        scenarioName: z.ZodString;
+        modelConfig: z.ZodOptional<z.ZodObject<{
+            model: z.ZodEnum<typeof import("../index.js").ModelIds>;
+            temperature: z.ZodOptional<z.ZodNumber>;
+            maxTokens: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+        assertionResults: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            assertionId: z.ZodString;
+            assertionType: z.ZodString;
+            assertionName: z.ZodString;
+            status: z.ZodEnum<typeof import("./eval-result.js").AssertionResultStatus>;
+            message: z.ZodOptional<z.ZodString>;
+            expected: z.ZodOptional<z.ZodString>;
+            actual: z.ZodOptional<z.ZodString>;
+            duration: z.ZodOptional<z.ZodNumber>;
+            details: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            llmTraceSteps: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                stepNumber: z.ZodNumber;
+                type: z.ZodEnum<typeof import("./metrics.js").LLMStepType>;
+                model: z.ZodString;
+                provider: z.ZodString;
+                startedAt: z.ZodString;
+                durationMs: z.ZodNumber;
+                tokenUsage: z.ZodObject<{
+                    prompt: z.ZodNumber;
+                    completion: z.ZodNumber;
+                    total: z.ZodNumber;
+                }, z.core.$strip>;
+                costUsd: z.ZodNumber;
+                toolName: z.ZodOptional<z.ZodString>;
+                toolArguments: z.ZodOptional<z.ZodString>;
+                inputPreview: z.ZodOptional<z.ZodString>;
+                outputPreview: z.ZodOptional<z.ZodString>;
+                success: z.ZodBoolean;
+                error: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+        }, z.core.$strip>>;
+        metrics: z.ZodOptional<z.ZodObject<{
+            totalAssertions: z.ZodNumber;
+            passed: z.ZodNumber;
+            failed: z.ZodNumber;
+            skipped: z.ZodNumber;
+            errors: z.ZodNumber;
+            passRate: z.ZodNumber;
+            avgDuration: z.ZodNumber;
+            totalDuration: z.ZodNumber;
+        }, z.core.$strip>>;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        passRate: z.ZodNumber;
+        duration: z.ZodNumber;
+        outputText: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            path: z.ZodString;
+            content: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        startedAt: z.ZodOptional<z.ZodString>;
+        completedAt: z.ZodOptional<z.ZodString>;
+        llmTrace: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            steps: z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                stepNumber: z.ZodNumber;
+                type: z.ZodEnum<typeof import("./metrics.js").LLMStepType>;
+                model: z.ZodString;
+                provider: z.ZodString;
+                startedAt: z.ZodString;
+                durationMs: z.ZodNumber;
+                tokenUsage: z.ZodObject<{
+                    prompt: z.ZodNumber;
+                    completion: z.ZodNumber;
+                    total: z.ZodNumber;
+                }, z.core.$strip>;
+                costUsd: z.ZodNumber;
+                toolName: z.ZodOptional<z.ZodString>;
+                toolArguments: z.ZodOptional<z.ZodString>;
+                inputPreview: z.ZodOptional<z.ZodString>;
+                outputPreview: z.ZodOptional<z.ZodString>;
+                success: z.ZodBoolean;
+                error: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>;
+            summary: z.ZodObject<{
+                totalSteps: z.ZodNumber;
+                totalDurationMs: z.ZodNumber;
+                totalTokens: z.ZodObject<{
+                    prompt: z.ZodNumber;
+                    completion: z.ZodNumber;
+                    total: z.ZodNumber;
+                }, z.core.$strip>;
+                totalCostUsd: z.ZodNumber;
+                stepTypeBreakdown: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+                    count: z.ZodNumber;
+                    durationMs: z.ZodNumber;
+                    tokens: z.ZodNumber;
+                    costUsd: z.ZodNumber;
+                }, z.core.$strip>>>;
+                modelBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
+                    count: z.ZodNumber;
+                    durationMs: z.ZodNumber;
+                    tokens: z.ZodNumber;
+                    costUsd: z.ZodNumber;
+                }, z.core.$strip>>;
+                modelsUsed: z.ZodArray<z.ZodString>;
+            }, z.core.$strip>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    aggregateMetrics: z.ZodObject<{
+        totalAssertions: z.ZodNumber;
+        passed: z.ZodNumber;
+        failed: z.ZodNumber;
+        skipped: z.ZodNumber;
+        errors: z.ZodNumber;
+        passRate: z.ZodNumber;
+        avgDuration: z.ZodNumber;
+        totalDuration: z.ZodNumber;
+    }, z.core.$strip>;
+    failureAnalyses: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        category: z.ZodEnum<typeof FailureCategory>;
+        severity: z.ZodEnum<typeof FailureSeverity>;
+        summary: z.ZodString;
+        details: z.ZodString;
+        rootCause: z.ZodString;
+        suggestedFix: z.ZodString;
+        relatedAssertions: z.ZodArray<z.ZodString>;
+        codeSnippet: z.ZodOptional<z.ZodString>;
+        similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        patternId: z.ZodOptional<z.ZodString>;
+        diff: z.ZodOptional<z.ZodObject<{
+            path: z.ZodString;
+            expected: z.ZodString;
+            actual: z.ZodString;
+            diffLines: z.ZodArray<z.ZodObject<{
+                type: z.ZodEnum<{
+                    added: "added";
+                    removed: "removed";
+                    unchanged: "unchanged";
+                }>;
+                content: z.ZodString;
+                lineNumber: z.ZodNumber;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        executionTrace: z.ZodOptional<z.ZodObject<{
+            commands: z.ZodArray<z.ZodObject<{
+                command: z.ZodString;
+                exitCode: z.ZodNumber;
+                output: z.ZodOptional<z.ZodString>;
+                duration: z.ZodNumber;
+            }, z.core.$strip>>;
+            filesModified: z.ZodArray<z.ZodObject<{
+                path: z.ZodString;
+                action: z.ZodEnum<{
+                    deleted: "deleted";
+                    created: "created";
+                    modified: "modified";
+                }>;
+            }, z.core.$strip>>;
+            apiCalls: z.ZodArray<z.ZodObject<{
+                endpoint: z.ZodString;
+                tokensUsed: z.ZodNumber;
+                duration: z.ZodNumber;
+            }, z.core.$strip>>;
+            totalDuration: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>>;
+    llmTraceSummary: z.ZodOptional<z.ZodObject<{
+        totalSteps: z.ZodNumber;
+        totalDurationMs: z.ZodNumber;
+        totalTokens: z.ZodObject<{
+            prompt: z.ZodNumber;
+            completion: z.ZodNumber;
+            total: z.ZodNumber;
+        }, z.core.$strip>;
+        totalCostUsd: z.ZodNumber;
+        stepTypeBreakdown: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+            count: z.ZodNumber;
+            durationMs: z.ZodNumber;
+            tokens: z.ZodNumber;
+            costUsd: z.ZodNumber;
+        }, z.core.$strip>>>;
+        modelBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
+            count: z.ZodNumber;
+            durationMs: z.ZodNumber;
+            tokens: z.ZodNumber;
+            costUsd: z.ZodNumber;
+        }, z.core.$strip>>;
+        modelsUsed: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>>;
+    trigger: z.ZodOptional<z.ZodObject<{
+        id: z.ZodString;
+        metadata: z.ZodOptional<z.ZodObject<{
+            version: z.ZodOptional<z.ZodString>;
+            resourceUpdated: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>;
+        type: z.ZodEnum<typeof TriggerType>;
+    }, z.core.$strip>>;
+    startedAt: z.ZodOptional<z.ZodString>;
+    completedAt: z.ZodOptional<z.ZodString>;
+    liveTraceEvents: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        evalRunId: z.ZodString;
+        scenarioId: z.ZodString;
+        scenarioName: z.ZodString;
+        targetId: z.ZodString;
+        targetName: z.ZodString;
+        stepNumber: z.ZodNumber;
+        type: z.ZodEnum<typeof import("./live-trace.js").LiveTraceEventType>;
+        toolName: z.ZodOptional<z.ZodString>;
+        toolArgs: z.ZodOptional<z.ZodString>;
+        outputPreview: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodString;
+        isComplete: z.ZodBoolean;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+export type EvalRun = z.infer<typeof EvalRunSchema>;
+/**
+ * Input schema for creating a new EvalRun.
+ */
+export declare const CreateEvalRunInputSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodString;
+    deleted: z.ZodOptional<z.ZodBoolean>;
+    projectId: z.ZodString;
+    scenarioIds: z.ZodArray<z.ZodString>;
+    agentId: z.ZodOptional<z.ZodString>;
+    skillsGroupId: z.ZodOptional<z.ZodString>;
+    failureAnalyses: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        category: z.ZodEnum<typeof FailureCategory>;
+        severity: z.ZodEnum<typeof FailureSeverity>;
+        summary: z.ZodString;
+        details: z.ZodString;
+        rootCause: z.ZodString;
+        suggestedFix: z.ZodString;
+        relatedAssertions: z.ZodArray<z.ZodString>;
+        codeSnippet: z.ZodOptional<z.ZodString>;
+        similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        patternId: z.ZodOptional<z.ZodString>;
+        diff: z.ZodOptional<z.ZodObject<{
+            path: z.ZodString;
+            expected: z.ZodString;
+            actual: z.ZodString;
+            diffLines: z.ZodArray<z.ZodObject<{
+                type: z.ZodEnum<{
+                    added: "added";
+                    removed: "removed";
+                    unchanged: "unchanged";
+                }>;
+                content: z.ZodString;
+                lineNumber: z.ZodNumber;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        executionTrace: z.ZodOptional<z.ZodObject<{
+            commands: z.ZodArray<z.ZodObject<{
+                command: z.ZodString;
+                exitCode: z.ZodNumber;
+                output: z.ZodOptional<z.ZodString>;
+                duration: z.ZodNumber;
+            }, z.core.$strip>>;
+            filesModified: z.ZodArray<z.ZodObject<{
+                path: z.ZodString;
+                action: z.ZodEnum<{
+                    deleted: "deleted";
+                    created: "created";
+                    modified: "modified";
+                }>;
+            }, z.core.$strip>>;
+            apiCalls: z.ZodArray<z.ZodObject<{
+                endpoint: z.ZodString;
+                tokensUsed: z.ZodNumber;
+                duration: z.ZodNumber;
+            }, z.core.$strip>>;
+            totalDuration: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>>;
+    llmTraceSummary: z.ZodOptional<z.ZodObject<{
+        totalSteps: z.ZodNumber;
+        totalDurationMs: z.ZodNumber;
+        totalTokens: z.ZodObject<{
+            prompt: z.ZodNumber;
+            completion: z.ZodNumber;
+            total: z.ZodNumber;
+        }, z.core.$strip>;
+        totalCostUsd: z.ZodNumber;
+        stepTypeBreakdown: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+            count: z.ZodNumber;
+            durationMs: z.ZodNumber;
+            tokens: z.ZodNumber;
+            costUsd: z.ZodNumber;
+        }, z.core.$strip>>>;
+        modelBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
+            count: z.ZodNumber;
+            durationMs: z.ZodNumber;
+            tokens: z.ZodNumber;
+            costUsd: z.ZodNumber;
+        }, z.core.$strip>>;
+        modelsUsed: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>>;
+    trigger: z.ZodOptional<z.ZodObject<{
+        id: z.ZodString;
+        metadata: z.ZodOptional<z.ZodObject<{
+            version: z.ZodOptional<z.ZodString>;
+            resourceUpdated: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>;
+        type: z.ZodEnum<typeof TriggerType>;
+    }, z.core.$strip>>;
+    liveTraceEvents: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        evalRunId: z.ZodString;
+        scenarioId: z.ZodString;
+        scenarioName: z.ZodString;
+        targetId: z.ZodString;
+        targetName: z.ZodString;
+        stepNumber: z.ZodNumber;
+        type: z.ZodEnum<typeof import("./live-trace.js").LiveTraceEventType>;
+        toolName: z.ZodOptional<z.ZodString>;
+        toolArgs: z.ZodOptional<z.ZodString>;
+        outputPreview: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodString;
+        isComplete: z.ZodBoolean;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+export type CreateEvalRunInput = z.infer<typeof CreateEvalRunInputSchema>;
+/**
+ * Evaluation progress schema.
+ */
+export declare const EvaluationProgressSchema: z.ZodObject<{
+    runId: z.ZodString;
+    targetId: z.ZodString;
+    totalScenarios: z.ZodNumber;
+    completedScenarios: z.ZodNumber;
+    scenarioProgress: z.ZodArray<z.ZodObject<{
+        scenarioId: z.ZodString;
+        currentStep: z.ZodString;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    createdAt: z.ZodNumber;
+}, z.core.$strip>;
+export type EvaluationProgress = z.infer<typeof EvaluationProgressSchema>;
+/**
+ * Evaluation log schema.
+ */
+export declare const EvaluationLogSchema: z.ZodObject<{
+    runId: z.ZodString;
+    scenarioId: z.ZodString;
+    log: z.ZodObject<{
+        level: z.ZodEnum<{
+            error: "error";
+            info: "info";
+            debug: "debug";
+        }>;
+        message: z.ZodOptional<z.ZodString>;
+        args: z.ZodOptional<z.ZodArray<z.ZodAny>>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export type EvaluationLog = z.infer<typeof EvaluationLogSchema>;
+/**
+ * LLM timeout constant (2 minutes).
+ */
+export declare const LLM_TIMEOUT = 120000;

package/build/types/evaluation/index.d.ts

@@ -0,0 +1,4 @@
+export * from './metrics.js';
+export * from './eval-result.js';
+export * from './eval-run.js';
+export * from './live-trace.js';

package/build/types/evaluation/live-trace.d.ts

@@ -0,0 +1,47 @@
+import { z } from 'zod';
+/**
+ * Live trace event type enum.
+ * Maps to the step types but includes streaming states.
+ */
+export declare enum LiveTraceEventType {
+    THINKING = "thinking",
+    TOOL_USE = "tool_use",
+    COMPLETION = "completion",
+    TOOL_RESULT = "tool_result"
+}
+/**
+ * Live trace event schema.
+ * Represents a single trace event emitted during agent execution.
+ */
+export declare const LiveTraceEventSchema: z.ZodObject<{
+    evalRunId: z.ZodString;
+    scenarioId: z.ZodString;
+    scenarioName: z.ZodString;
+    targetId: z.ZodString;
+    targetName: z.ZodString;
+    stepNumber: z.ZodNumber;
+    type: z.ZodEnum<typeof LiveTraceEventType>;
+    toolName: z.ZodOptional<z.ZodString>;
+    toolArgs: z.ZodOptional<z.ZodString>;
+    outputPreview: z.ZodOptional<z.ZodString>;
+    timestamp: z.ZodString;
+    isComplete: z.ZodBoolean;
+}, z.core.$strip>;
+export type LiveTraceEvent = z.infer<typeof LiveTraceEventSchema>;
+/**
+ * Prefix used in stdout to identify trace events.
+ * Format: TRACE_EVENT:{json}
+ */
+export declare const TRACE_EVENT_PREFIX = "TRACE_EVENT:";
+/**
+ * Parse a line from stdout to extract a trace event if present.
+ * @param line - A line from stdout
+ * @returns The parsed LiveTraceEvent or null if not a trace event line
+ */
+export declare function parseTraceEventLine(line: string): LiveTraceEvent | null;
+/**
+ * Format a trace event as a stdout line.
+ * @param event - The trace event to format
+ * @returns The formatted line with prefix
+ */
+export declare function formatTraceEventLine(event: LiveTraceEvent): string;