@wix/evalforge-types 0.4.0 → 0.6.0

package/build/index.js

@@ -634,128 +634,46 @@ var LLMTraceSchema = import_zod21.z.object({
 });
 
 // src/evaluation/eval-result.ts
-var import_zod22 = require("zod");
-var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
-  AssertionResultStatus2["PASSED"] = "passed";
-  AssertionResultStatus2["FAILED"] = "failed";
-  AssertionResultStatus2["SKIPPED"] = "skipped";
-  AssertionResultStatus2["ERROR"] = "error";
-  return AssertionResultStatus2;
-})(AssertionResultStatus || {});
-var AssertionResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  assertionId: import_zod22.z.string(),
-  assertionType: import_zod22.z.string(),
-  assertionName: import_zod22.z.string(),
-  status: import_zod22.z.enum(AssertionResultStatus),
-  message: import_zod22.z.string().optional(),
-  expected: import_zod22.z.string().optional(),
-  actual: import_zod22.z.string().optional(),
-  duration: import_zod22.z.number().optional(),
-  details: import_zod22.z.record(import_zod22.z.string(), import_zod22.z.unknown()).optional(),
-  llmTraceSteps: import_zod22.z.array(LLMTraceStepSchema).optional()
-});
-var EvalRunResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  targetId: import_zod22.z.string(),
-  targetName: import_zod22.z.string().optional(),
-  scenarioId: import_zod22.z.string(),
-  scenarioName: import_zod22.z.string(),
-  modelConfig: ModelConfigSchema.optional(),
-  assertionResults: import_zod22.z.array(AssertionResultSchema),
-  metrics: EvalMetricsSchema.optional(),
-  passed: import_zod22.z.number(),
-  failed: import_zod22.z.number(),
-  passRate: import_zod22.z.number(),
-  duration: import_zod22.z.number(),
-  outputText: import_zod22.z.string().optional(),
-  files: import_zod22.z.array(ExpectedFileSchema).optional(),
-  startedAt: import_zod22.z.string().optional(),
-  completedAt: import_zod22.z.string().optional(),
-  llmTrace: LLMTraceSchema.optional()
-});
-var PromptResultSchema = import_zod22.z.object({
-  text: import_zod22.z.string(),
-  files: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  finishReason: import_zod22.z.string().optional(),
-  reasoning: import_zod22.z.string().optional(),
-  reasoningDetails: import_zod22.z.unknown().optional(),
-  toolCalls: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  toolResults: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  warnings: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  sources: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  steps: import_zod22.z.array(import_zod22.z.unknown()),
-  generationTimeMs: import_zod22.z.number(),
-  prompt: import_zod22.z.string(),
-  systemPrompt: import_zod22.z.string(),
-  usage: import_zod22.z.object({
-    totalTokens: import_zod22.z.number().optional(),
-    totalMicrocentsSpent: import_zod22.z.number().optional()
-  })
-});
-var EvaluationResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  runId: import_zod22.z.string(),
-  timestamp: import_zod22.z.number(),
-  promptResult: PromptResultSchema,
-  testResults: import_zod22.z.array(import_zod22.z.unknown()),
-  tags: import_zod22.z.array(import_zod22.z.string()).optional(),
-  feedback: import_zod22.z.string().optional(),
-  score: import_zod22.z.number(),
-  suiteId: import_zod22.z.string().optional()
-});
-var LeanEvaluationResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  runId: import_zod22.z.string(),
-  timestamp: import_zod22.z.number(),
-  tags: import_zod22.z.array(import_zod22.z.string()).optional(),
-  scenarioId: import_zod22.z.string(),
-  scenarioVersion: import_zod22.z.number().optional(),
-  targetId: import_zod22.z.string(),
-  targetVersion: import_zod22.z.number().optional(),
-  suiteId: import_zod22.z.string().optional(),
-  score: import_zod22.z.number(),
-  time: import_zod22.z.number().optional(),
-  microcentsSpent: import_zod22.z.number().optional()
-});
+var import_zod24 = require("zod");
 
 // src/evaluation/eval-run.ts
-var import_zod24 = require("zod");
+var import_zod23 = require("zod");
 
 // src/evaluation/live-trace.ts
-var import_zod23 = require("zod");
+var import_zod22 = require("zod");
 var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
   LiveTraceEventType2["THINKING"] = "thinking";
   LiveTraceEventType2["TOOL_USE"] = "tool_use";
   LiveTraceEventType2["COMPLETION"] = "completion";
   LiveTraceEventType2["TOOL_RESULT"] = "tool_result";
+  LiveTraceEventType2["DIAGNOSTIC"] = "diagnostic";
   return LiveTraceEventType2;
 })(LiveTraceEventType || {});
-var LiveTraceEventSchema = import_zod23.z.object({
+var LiveTraceEventSchema = import_zod22.z.object({
   /** The evaluation run ID */
-  evalRunId: import_zod23.z.string(),
+  evalRunId: import_zod22.z.string(),
   /** The scenario ID being executed */
-  scenarioId: import_zod23.z.string(),
+  scenarioId: import_zod22.z.string(),
   /** The scenario name for display */
-  scenarioName: import_zod23.z.string(),
+  scenarioName: import_zod22.z.string(),
   /** The target ID (skill, agent, etc.) */
-  targetId: import_zod23.z.string(),
+  targetId: import_zod22.z.string(),
   /** The target name for display */
-  targetName: import_zod23.z.string(),
+  targetName: import_zod22.z.string(),
   /** Step number in the current scenario execution */
-  stepNumber: import_zod23.z.number(),
+  stepNumber: import_zod22.z.number(),
   /** Type of trace event */
-  type: import_zod23.z.enum(LiveTraceEventType),
+  type: import_zod22.z.enum(LiveTraceEventType),
   /** Tool name if this is a tool_use event */
-  toolName: import_zod23.z.string().optional(),
+  toolName: import_zod22.z.string().optional(),
   /** Tool arguments preview (truncated JSON) */
-  toolArgs: import_zod23.z.string().optional(),
+  toolArgs: import_zod22.z.string().optional(),
   /** Output preview (truncated text) */
-  outputPreview: import_zod23.z.string().optional(),
+  outputPreview: import_zod22.z.string().optional(),
   /** Timestamp when this event occurred */
-  timestamp: import_zod23.z.string(),
+  timestamp: import_zod22.z.string(),
   /** Whether this is the final event for this scenario */
-  isComplete: import_zod23.z.boolean()
+  isComplete: import_zod22.z.boolean()
 });
 var TRACE_EVENT_PREFIX = "TRACE_EVENT:";
 function parseTraceEventLine(line) {
@@ -783,14 +701,14 @@ var TriggerType = /* @__PURE__ */ ((TriggerType2) => {
   TriggerType2["MANUAL"] = "MANUAL";
   return TriggerType2;
 })(TriggerType || {});
-var TriggerMetadataSchema = import_zod24.z.object({
-  version: import_zod24.z.string().optional(),
-  resourceUpdated: import_zod24.z.array(import_zod24.z.string()).optional()
+var TriggerMetadataSchema = import_zod23.z.object({
+  version: import_zod23.z.string().optional(),
+  resourceUpdated: import_zod23.z.array(import_zod23.z.string()).optional()
 });
-var TriggerSchema = import_zod24.z.object({
-  id: import_zod24.z.string(),
+var TriggerSchema = import_zod23.z.object({
+  id: import_zod23.z.string(),
   metadata: TriggerMetadataSchema.optional(),
-  type: import_zod24.z.enum(TriggerType)
+  type: import_zod23.z.enum(TriggerType)
 });
 var FailureCategory = /* @__PURE__ */ ((FailureCategory2) => {
   FailureCategory2["MISSING_FILE"] = "missing_file";
@@ -808,89 +726,89 @@ var FailureSeverity = /* @__PURE__ */ ((FailureSeverity2) => {
   FailureSeverity2["LOW"] = "low";
   return FailureSeverity2;
 })(FailureSeverity || {});
-var DiffLineTypeSchema = import_zod24.z.enum(["added", "removed", "unchanged"]);
-var DiffLineSchema = import_zod24.z.object({
+var DiffLineTypeSchema = import_zod23.z.enum(["added", "removed", "unchanged"]);
+var DiffLineSchema = import_zod23.z.object({
   type: DiffLineTypeSchema,
-  content: import_zod24.z.string(),
-  lineNumber: import_zod24.z.number()
-});
-var DiffContentSchema = import_zod24.z.object({
-  path: import_zod24.z.string(),
-  expected: import_zod24.z.string(),
-  actual: import_zod24.z.string(),
-  diffLines: import_zod24.z.array(DiffLineSchema)
-});
-var CommandExecutionSchema = import_zod24.z.object({
-  command: import_zod24.z.string(),
-  exitCode: import_zod24.z.number(),
-  output: import_zod24.z.string().optional(),
-  duration: import_zod24.z.number()
-});
-var FileModificationSchema = import_zod24.z.object({
-  path: import_zod24.z.string(),
-  action: import_zod24.z.enum(["created", "modified", "deleted"])
-});
-var ApiCallSchema = import_zod24.z.object({
-  endpoint: import_zod24.z.string(),
-  tokensUsed: import_zod24.z.number(),
-  duration: import_zod24.z.number()
-});
-var ExecutionTraceSchema = import_zod24.z.object({
-  commands: import_zod24.z.array(CommandExecutionSchema),
-  filesModified: import_zod24.z.array(FileModificationSchema),
-  apiCalls: import_zod24.z.array(ApiCallSchema),
-  totalDuration: import_zod24.z.number()
-});
-var FailureAnalysisSchema = import_zod24.z.object({
-  category: import_zod24.z.enum(FailureCategory),
-  severity: import_zod24.z.enum(FailureSeverity),
-  summary: import_zod24.z.string(),
-  details: import_zod24.z.string(),
-  rootCause: import_zod24.z.string(),
-  suggestedFix: import_zod24.z.string(),
-  relatedAssertions: import_zod24.z.array(import_zod24.z.string()),
-  codeSnippet: import_zod24.z.string().optional(),
-  similarIssues: import_zod24.z.array(import_zod24.z.string()).optional(),
-  patternId: import_zod24.z.string().optional(),
+  content: import_zod23.z.string(),
+  lineNumber: import_zod23.z.number()
+});
+var DiffContentSchema = import_zod23.z.object({
+  path: import_zod23.z.string(),
+  expected: import_zod23.z.string(),
+  actual: import_zod23.z.string(),
+  diffLines: import_zod23.z.array(DiffLineSchema)
+});
+var CommandExecutionSchema = import_zod23.z.object({
+  command: import_zod23.z.string(),
+  exitCode: import_zod23.z.number(),
+  output: import_zod23.z.string().optional(),
+  duration: import_zod23.z.number()
+});
+var FileModificationSchema = import_zod23.z.object({
+  path: import_zod23.z.string(),
+  action: import_zod23.z.enum(["created", "modified", "deleted"])
+});
+var ApiCallSchema = import_zod23.z.object({
+  endpoint: import_zod23.z.string(),
+  tokensUsed: import_zod23.z.number(),
+  duration: import_zod23.z.number()
+});
+var ExecutionTraceSchema = import_zod23.z.object({
+  commands: import_zod23.z.array(CommandExecutionSchema),
+  filesModified: import_zod23.z.array(FileModificationSchema),
+  apiCalls: import_zod23.z.array(ApiCallSchema),
+  totalDuration: import_zod23.z.number()
+});
+var FailureAnalysisSchema = import_zod23.z.object({
+  category: import_zod23.z.enum(FailureCategory),
+  severity: import_zod23.z.enum(FailureSeverity),
+  summary: import_zod23.z.string(),
+  details: import_zod23.z.string(),
+  rootCause: import_zod23.z.string(),
+  suggestedFix: import_zod23.z.string(),
+  relatedAssertions: import_zod23.z.array(import_zod23.z.string()),
+  codeSnippet: import_zod23.z.string().optional(),
+  similarIssues: import_zod23.z.array(import_zod23.z.string()).optional(),
+  patternId: import_zod23.z.string().optional(),
   // Extended fields for detailed debugging
   diff: DiffContentSchema.optional(),
   executionTrace: ExecutionTraceSchema.optional()
 });
 var EvalRunSchema = TenantEntitySchema.extend({
   /** Agent ID for this run */
-  agentId: import_zod24.z.string().optional(),
+  agentId: import_zod23.z.string().optional(),
   /** Skills group ID for this run */
-  skillsGroupId: import_zod24.z.string().optional(),
+  skillsGroupId: import_zod23.z.string().optional(),
   /** Scenario IDs to run */
-  scenarioIds: import_zod24.z.array(import_zod24.z.string()),
+  scenarioIds: import_zod23.z.array(import_zod23.z.string()),
   /** Current status */
   status: EvalStatusSchema,
   /** Progress percentage (0-100) */
-  progress: import_zod24.z.number(),
+  progress: import_zod23.z.number(),
   /** Results for each scenario/target combination */
-  results: import_zod24.z.array(EvalRunResultSchema),
+  results: import_zod23.z.array(EvalRunResultSchema),
   /** Aggregated metrics across all results */
   aggregateMetrics: EvalMetricsSchema,
   /** Failure analyses */
-  failureAnalyses: import_zod24.z.array(FailureAnalysisSchema).optional(),
+  failureAnalyses: import_zod23.z.array(FailureAnalysisSchema).optional(),
   /** Aggregated LLM trace summary */
   llmTraceSummary: LLMTraceSummarySchema.optional(),
   /** What triggered this run */
   trigger: TriggerSchema.optional(),
   /** When the run started (set when evaluation is triggered) */
-  startedAt: import_zod24.z.string().optional(),
+  startedAt: import_zod23.z.string().optional(),
   /** When the run completed */
-  completedAt: import_zod24.z.string().optional(),
+  completedAt: import_zod23.z.string().optional(),
   /** Live trace events captured during execution (for playback on results page) */
-  liveTraceEvents: import_zod24.z.array(LiveTraceEventSchema).optional(),
+  liveTraceEvents: import_zod23.z.array(LiveTraceEventSchema).optional(),
   /** Remote job ID for tracking execution in Dev Machines */
-  jobId: import_zod24.z.string().optional(),
+  jobId: import_zod23.z.string().optional(),
   /** Remote job status from the Dev Machine API (PENDING, RUNNING, COMPLETED, FAILED, CANCELLED) */
-  jobStatus: import_zod24.z.string().optional(),
+  jobStatus: import_zod23.z.string().optional(),
   /** Remote job error message if the job failed */
-  jobError: import_zod24.z.string().optional(),
+  jobError: import_zod23.z.string().optional(),
   /** Timestamp of the last job status check */
-  jobStatusCheckedAt: import_zod24.z.string().optional()
+  jobStatusCheckedAt: import_zod23.z.string().optional()
 });
 var CreateEvalRunInputSchema = EvalRunSchema.omit({
   id: true,
@@ -903,32 +821,119 @@ var CreateEvalRunInputSchema = EvalRunSchema.omit({
   startedAt: true,
   completedAt: true
 });
-var EvaluationProgressSchema = import_zod24.z.object({
-  runId: import_zod24.z.string(),
-  targetId: import_zod24.z.string(),
-  totalScenarios: import_zod24.z.number(),
-  completedScenarios: import_zod24.z.number(),
-  scenarioProgress: import_zod24.z.array(
-    import_zod24.z.object({
-      scenarioId: import_zod24.z.string(),
-      currentStep: import_zod24.z.string(),
-      error: import_zod24.z.string().optional()
+var EvaluationProgressSchema = import_zod23.z.object({
+  runId: import_zod23.z.string(),
+  targetId: import_zod23.z.string(),
+  totalScenarios: import_zod23.z.number(),
+  completedScenarios: import_zod23.z.number(),
+  scenarioProgress: import_zod23.z.array(
+    import_zod23.z.object({
+      scenarioId: import_zod23.z.string(),
+      currentStep: import_zod23.z.string(),
+      error: import_zod23.z.string().optional()
     })
   ),
-  createdAt: import_zod24.z.number()
+  createdAt: import_zod23.z.number()
 });
-var EvaluationLogSchema = import_zod24.z.object({
-  runId: import_zod24.z.string(),
-  scenarioId: import_zod24.z.string(),
-  log: import_zod24.z.object({
-    level: import_zod24.z.enum(["info", "error", "debug"]),
-    message: import_zod24.z.string().optional(),
-    args: import_zod24.z.array(import_zod24.z.any()).optional(),
-    error: import_zod24.z.string().optional()
+var EvaluationLogSchema = import_zod23.z.object({
+  runId: import_zod23.z.string(),
+  scenarioId: import_zod23.z.string(),
+  log: import_zod23.z.object({
+    level: import_zod23.z.enum(["info", "error", "debug"]),
+    message: import_zod23.z.string().optional(),
+    args: import_zod23.z.array(import_zod23.z.any()).optional(),
+    error: import_zod23.z.string().optional()
   })
 });
 var LLM_TIMEOUT = 12e4;
 
+// src/evaluation/eval-result.ts
+var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
+  AssertionResultStatus2["PASSED"] = "passed";
+  AssertionResultStatus2["FAILED"] = "failed";
+  AssertionResultStatus2["SKIPPED"] = "skipped";
+  AssertionResultStatus2["ERROR"] = "error";
+  return AssertionResultStatus2;
+})(AssertionResultStatus || {});
+var AssertionResultSchema = import_zod24.z.object({
+  id: import_zod24.z.string(),
+  assertionId: import_zod24.z.string(),
+  assertionType: import_zod24.z.string(),
+  assertionName: import_zod24.z.string(),
+  status: import_zod24.z.enum(AssertionResultStatus),
+  message: import_zod24.z.string().optional(),
+  expected: import_zod24.z.string().optional(),
+  actual: import_zod24.z.string().optional(),
+  duration: import_zod24.z.number().optional(),
+  details: import_zod24.z.record(import_zod24.z.string(), import_zod24.z.unknown()).optional(),
+  llmTraceSteps: import_zod24.z.array(LLMTraceStepSchema).optional()
+});
+var EvalRunResultSchema = import_zod24.z.object({
+  id: import_zod24.z.string(),
+  targetId: import_zod24.z.string(),
+  targetName: import_zod24.z.string().optional(),
+  scenarioId: import_zod24.z.string(),
+  scenarioName: import_zod24.z.string(),
+  modelConfig: ModelConfigSchema.optional(),
+  assertionResults: import_zod24.z.array(AssertionResultSchema),
+  metrics: EvalMetricsSchema.optional(),
+  passed: import_zod24.z.number(),
+  failed: import_zod24.z.number(),
+  passRate: import_zod24.z.number(),
+  duration: import_zod24.z.number(),
+  outputText: import_zod24.z.string().optional(),
+  files: import_zod24.z.array(ExpectedFileSchema).optional(),
+  /** File diffs showing changes made by the agent during execution */
+  fileDiffs: import_zod24.z.array(DiffContentSchema).optional(),
+  startedAt: import_zod24.z.string().optional(),
+  completedAt: import_zod24.z.string().optional(),
+  llmTrace: LLMTraceSchema.optional()
+});
+var PromptResultSchema = import_zod24.z.object({
+  text: import_zod24.z.string(),
+  files: import_zod24.z.array(import_zod24.z.unknown()).optional(),
+  finishReason: import_zod24.z.string().optional(),
+  reasoning: import_zod24.z.string().optional(),
+  reasoningDetails: import_zod24.z.unknown().optional(),
+  toolCalls: import_zod24.z.array(import_zod24.z.unknown()).optional(),
+  toolResults: import_zod24.z.array(import_zod24.z.unknown()).optional(),
+  warnings: import_zod24.z.array(import_zod24.z.unknown()).optional(),
+  sources: import_zod24.z.array(import_zod24.z.unknown()).optional(),
+  steps: import_zod24.z.array(import_zod24.z.unknown()),
+  generationTimeMs: import_zod24.z.number(),
+  prompt: import_zod24.z.string(),
+  systemPrompt: import_zod24.z.string(),
+  usage: import_zod24.z.object({
+    totalTokens: import_zod24.z.number().optional(),
+    totalMicrocentsSpent: import_zod24.z.number().optional()
+  })
+});
+var EvaluationResultSchema = import_zod24.z.object({
+  id: import_zod24.z.string(),
+  runId: import_zod24.z.string(),
+  timestamp: import_zod24.z.number(),
+  promptResult: PromptResultSchema,
+  testResults: import_zod24.z.array(import_zod24.z.unknown()),
+  tags: import_zod24.z.array(import_zod24.z.string()).optional(),
+  feedback: import_zod24.z.string().optional(),
+  score: import_zod24.z.number(),
+  suiteId: import_zod24.z.string().optional()
+});
+var LeanEvaluationResultSchema = import_zod24.z.object({
+  id: import_zod24.z.string(),
+  runId: import_zod24.z.string(),
+  timestamp: import_zod24.z.number(),
+  tags: import_zod24.z.array(import_zod24.z.string()).optional(),
+  scenarioId: import_zod24.z.string(),
+  scenarioVersion: import_zod24.z.number().optional(),
+  targetId: import_zod24.z.string(),
+  targetVersion: import_zod24.z.number().optional(),
+  suiteId: import_zod24.z.string().optional(),
+  score: import_zod24.z.number(),
+  time: import_zod24.z.number().optional(),
+  microcentsSpent: import_zod24.z.number().optional()
+});
+
 // src/project/project.ts
 var import_zod25 = require("zod");
 var ProjectSchema = BaseEntitySchema.extend({