@wix/evalforge-types 0.5.0 → 0.7.0

package/build/index.js

@@ -27,9 +27,11 @@ __export(index_exports, {
   ApiCallSchema: () => ApiCallSchema,
   AssertionResultSchema: () => AssertionResultSchema,
   AssertionResultStatus: () => AssertionResultStatus,
+  AssertionSchema: () => AssertionSchema,
   BaseEntitySchema: () => BaseEntitySchema,
   BaseTestSchema: () => BaseTestSchema,
   BuildCheckTestSchema: () => BuildCheckTestSchema,
+  BuildPassedAssertionSchema: () => BuildPassedAssertionSchema,
   CommandExecutionSchema: () => CommandExecutionSchema,
   CommandExecutionTestSchema: () => CommandExecutionTestSchema,
   CreateAgentInputSchema: () => CreateAgentInputSchema,
@@ -71,6 +73,7 @@ __export(index_exports, {
   LeanEvaluationResultSchema: () => LeanEvaluationResultSchema,
   LiveTraceEventSchema: () => LiveTraceEventSchema,
   LiveTraceEventType: () => LiveTraceEventType,
+  LlmJudgeAssertionSchema: () => LlmJudgeAssertionSchema,
   LocalProjectConfigSchema: () => LocalProjectConfigSchema,
   MCPServerConfigSchema: () => MCPServerConfigSchema,
   MetaSiteConfigSchema: () => MetaSiteConfigSchema,
@@ -86,6 +89,7 @@ __export(index_exports, {
   SkillMetadataSchema: () => SkillMetadataSchema,
   SkillSchema: () => SkillSchema,
   SkillVersionSchema: () => SkillVersionSchema,
+  SkillWasCalledAssertionSchema: () => SkillWasCalledAssertionSchema,
   SkillsGroupSchema: () => SkillsGroupSchema,
   TRACE_EVENT_PREFIX: () => TRACE_EVENT_PREFIX,
   TargetSchema: () => TargetSchema,
@@ -492,34 +496,67 @@ var TestSchema = import_zod17.z.discriminatedUnion("type", [
   PlaywrightNLTestSchema
 ]);
 
-// src/scenario/environment.ts
+// src/scenario/assertions.ts
 var import_zod18 = require("zod");
-var LocalProjectConfigSchema = import_zod18.z.object({
+var SkillWasCalledAssertionSchema = import_zod18.z.object({
+  type: import_zod18.z.literal("skill_was_called"),
+  /** Name of the skill that must have been called (matched against trace Skill tool args) */
+  skillName: import_zod18.z.string()
+});
+var BuildPassedAssertionSchema = import_zod18.z.object({
+  type: import_zod18.z.literal("build_passed"),
+  /** Command to run (default: "yarn build") */
+  command: import_zod18.z.string().optional(),
+  /** Expected exit code (default: 0) */
+  expectedExitCode: import_zod18.z.number().int().optional()
+});
+var LlmJudgeAssertionSchema = import_zod18.z.object({
+  type: import_zod18.z.literal("llm_judge"),
+  /** Prompt template; placeholders: {{output}}, {{cwd}}, {{changedFiles}}, {{trace}} */
+  prompt: import_zod18.z.string(),
+  /** Optional system prompt for the judge (default asks for JSON with score) */
+  systemPrompt: import_zod18.z.string().optional(),
+  /** Minimum score to pass (0–100, default 70) */
+  minScore: import_zod18.z.number().int().min(0).max(100).optional(),
+  /** Model for the judge (e.g. claude-3-5-haiku) */
+  model: import_zod18.z.string().optional(),
+  maxTokens: import_zod18.z.number().int().optional(),
+  temperature: import_zod18.z.number().min(0).max(1).optional()
+});
+var AssertionSchema = import_zod18.z.discriminatedUnion("type", [
+  SkillWasCalledAssertionSchema,
+  BuildPassedAssertionSchema,
+  LlmJudgeAssertionSchema
+]);
+
+// src/scenario/environment.ts
+var import_zod19 = require("zod");
+var LocalProjectConfigSchema = import_zod19.z.object({
   /** Template ID to use for the local project */
-  templateId: import_zod18.z.string().optional(),
+  templateId: import_zod19.z.string().optional(),
   /** Files to create in the project */
-  files: import_zod18.z.array(
-    import_zod18.z.object({
-      path: import_zod18.z.string().min(1),
-      content: import_zod18.z.string().min(1)
+  files: import_zod19.z.array(
+    import_zod19.z.object({
+      path: import_zod19.z.string().min(1),
+      content: import_zod19.z.string().min(1)
     })
   ).optional()
 });
-var MetaSiteConfigSchema = import_zod18.z.object({
-  configurations: import_zod18.z.array(
-    import_zod18.z.object({
-      name: import_zod18.z.string().min(1),
-      apiCalls: import_zod18.z.array(
-        import_zod18.z.object({
-          url: import_zod18.z.string().url(),
-          method: import_zod18.z.enum(["POST", "PUT"]),
-          body: import_zod18.z.string()
+var MetaSiteConfigSchema = import_zod19.z.object({
+  configurations: import_zod19.z.array(
+    import_zod19.z.object({
+      name: import_zod19.z.string().min(1),
+      apiCalls: import_zod19.z.array(
+        import_zod19.z.object({
+          url: import_zod19.z.string().url(),
+          method: import_zod19.z.enum(["POST", "PUT"]),
+          body: import_zod19.z.string()
         })
       )
     })
   ).optional()
 });
-var EnvironmentSchema = import_zod18.z.object({
+var EnvironmentSchema = import_zod19.z.object({
   /** Local project configuration */
   localProject: LocalProjectConfigSchema.optional(),
   /** Meta site configuration */
@@ -527,18 +564,20 @@ var EnvironmentSchema = import_zod18.z.object({
 });
 
 // src/scenario/test-scenario.ts
-var import_zod19 = require("zod");
-var ExpectedFileSchema = import_zod19.z.object({
+var import_zod20 = require("zod");
+var ExpectedFileSchema = import_zod20.z.object({
   /** Relative path where the file should be created */
-  path: import_zod19.z.string(),
+  path: import_zod20.z.string(),
   /** Optional expected content */
-  content: import_zod19.z.string().optional()
+  content: import_zod20.z.string().optional()
 });
 var TestScenarioSchema = TenantEntitySchema.extend({
   /** The prompt sent to the agent to trigger the task */
-  triggerPrompt: import_zod19.z.string().min(10),
+  triggerPrompt: import_zod20.z.string().min(10),
   /** ID of the template to use for this scenario */
-  templateId: import_zod19.z.string().optional()
+  templateId: import_zod20.z.string().optional(),
+  /** Assertions to evaluate for this scenario */
+  assertions: import_zod20.z.array(AssertionSchema).optional()
 });
 var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
   id: true,
@@ -549,10 +588,10 @@ var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
 var UpdateTestScenarioInputSchema = CreateTestScenarioInputSchema.partial();
 
 // src/suite/test-suite.ts
-var import_zod20 = require("zod");
+var import_zod21 = require("zod");
 var TestSuiteSchema = TenantEntitySchema.extend({
   /** IDs of test scenarios in this suite */
-  scenarioIds: import_zod20.z.array(import_zod20.z.string())
+  scenarioIds: import_zod21.z.array(import_zod21.z.string())
 });
 var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
   id: true,
@@ -563,21 +602,21 @@ var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
 var UpdateTestSuiteInputSchema = CreateTestSuiteInputSchema.partial();
 
 // src/evaluation/metrics.ts
-var import_zod21 = require("zod");
-var TokenUsageSchema = import_zod21.z.object({
-  prompt: import_zod21.z.number(),
-  completion: import_zod21.z.number(),
-  total: import_zod21.z.number()
-});
-var EvalMetricsSchema = import_zod21.z.object({
-  totalAssertions: import_zod21.z.number(),
-  passed: import_zod21.z.number(),
-  failed: import_zod21.z.number(),
-  skipped: import_zod21.z.number(),
-  errors: import_zod21.z.number(),
-  passRate: import_zod21.z.number(),
-  avgDuration: import_zod21.z.number(),
-  totalDuration: import_zod21.z.number()
+var import_zod22 = require("zod");
+var TokenUsageSchema = import_zod22.z.object({
+  prompt: import_zod22.z.number(),
+  completion: import_zod22.z.number(),
+  total: import_zod22.z.number()
+});
+var EvalMetricsSchema = import_zod22.z.object({
+  totalAssertions: import_zod22.z.number(),
+  passed: import_zod22.z.number(),
+  failed: import_zod22.z.number(),
+  skipped: import_zod22.z.number(),
+  errors: import_zod22.z.number(),
+  passRate: import_zod22.z.number(),
+  avgDuration: import_zod22.z.number(),
+  totalDuration: import_zod22.z.number()
 });
 var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
   EvalStatus2["PENDING"] = "pending";
@@ -587,7 +626,7 @@ var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
   EvalStatus2["CANCELLED"] = "cancelled";
   return EvalStatus2;
 })(EvalStatus || {});
-var EvalStatusSchema = import_zod21.z.enum(EvalStatus);
+var EvalStatusSchema = import_zod22.z.enum(EvalStatus);
 var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
   LLMStepType2["COMPLETION"] = "completion";
   LLMStepType2["TOOL_USE"] = "tool_use";
@@ -595,129 +634,46 @@ var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
   LLMStepType2["THINKING"] = "thinking";
   return LLMStepType2;
 })(LLMStepType || {});
-var LLMTraceStepSchema = import_zod21.z.object({
-  id: import_zod21.z.string(),
-  stepNumber: import_zod21.z.number(),
-  type: import_zod21.z.enum(LLMStepType),
-  model: import_zod21.z.string(),
-  provider: import_zod21.z.string(),
-  startedAt: import_zod21.z.string(),
-  durationMs: import_zod21.z.number(),
+var LLMTraceStepSchema = import_zod22.z.object({
+  id: import_zod22.z.string(),
+  stepNumber: import_zod22.z.number(),
+  type: import_zod22.z.enum(LLMStepType),
+  model: import_zod22.z.string(),
+  provider: import_zod22.z.string(),
+  startedAt: import_zod22.z.string(),
+  durationMs: import_zod22.z.number(),
   tokenUsage: TokenUsageSchema,
-  costUsd: import_zod21.z.number(),
-  toolName: import_zod21.z.string().optional(),
-  toolArguments: import_zod21.z.string().optional(),
-  inputPreview: import_zod21.z.string().optional(),
-  outputPreview: import_zod21.z.string().optional(),
-  success: import_zod21.z.boolean(),
-  error: import_zod21.z.string().optional()
-});
-var LLMBreakdownStatsSchema = import_zod21.z.object({
-  count: import_zod21.z.number(),
-  durationMs: import_zod21.z.number(),
-  tokens: import_zod21.z.number(),
-  costUsd: import_zod21.z.number()
-});
-var LLMTraceSummarySchema = import_zod21.z.object({
-  totalSteps: import_zod21.z.number(),
-  totalDurationMs: import_zod21.z.number(),
+  costUsd: import_zod22.z.number(),
+  toolName: import_zod22.z.string().optional(),
+  toolArguments: import_zod22.z.string().optional(),
+  inputPreview: import_zod22.z.string().optional(),
+  outputPreview: import_zod22.z.string().optional(),
+  success: import_zod22.z.boolean(),
+  error: import_zod22.z.string().optional()
+});
+var LLMBreakdownStatsSchema = import_zod22.z.object({
+  count: import_zod22.z.number(),
+  durationMs: import_zod22.z.number(),
+  tokens: import_zod22.z.number(),
+  costUsd: import_zod22.z.number()
+});
+var LLMTraceSummarySchema = import_zod22.z.object({
+  totalSteps: import_zod22.z.number(),
+  totalDurationMs: import_zod22.z.number(),
   totalTokens: TokenUsageSchema,
-  totalCostUsd: import_zod21.z.number(),
-  stepTypeBreakdown: import_zod21.z.record(import_zod21.z.string(), LLMBreakdownStatsSchema).optional(),
-  modelBreakdown: import_zod21.z.record(import_zod21.z.string(), LLMBreakdownStatsSchema),
-  modelsUsed: import_zod21.z.array(import_zod21.z.string())
-});
-var LLMTraceSchema = import_zod21.z.object({
-  id: import_zod21.z.string(),
-  steps: import_zod21.z.array(LLMTraceStepSchema),
+  totalCostUsd: import_zod22.z.number(),
+  stepTypeBreakdown: import_zod22.z.record(import_zod22.z.string(), LLMBreakdownStatsSchema).optional(),
+  modelBreakdown: import_zod22.z.record(import_zod22.z.string(), LLMBreakdownStatsSchema),
+  modelsUsed: import_zod22.z.array(import_zod22.z.string())
+});
+var LLMTraceSchema = import_zod22.z.object({
+  id: import_zod22.z.string(),
+  steps: import_zod22.z.array(LLMTraceStepSchema),
   summary: LLMTraceSummarySchema
 });
 
 // src/evaluation/eval-result.ts
-var import_zod22 = require("zod");
-var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
-  AssertionResultStatus2["PASSED"] = "passed";
-  AssertionResultStatus2["FAILED"] = "failed";
-  AssertionResultStatus2["SKIPPED"] = "skipped";
-  AssertionResultStatus2["ERROR"] = "error";
-  return AssertionResultStatus2;
-})(AssertionResultStatus || {});
-var AssertionResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  assertionId: import_zod22.z.string(),
-  assertionType: import_zod22.z.string(),
-  assertionName: import_zod22.z.string(),
-  status: import_zod22.z.enum(AssertionResultStatus),
-  message: import_zod22.z.string().optional(),
-  expected: import_zod22.z.string().optional(),
-  actual: import_zod22.z.string().optional(),
-  duration: import_zod22.z.number().optional(),
-  details: import_zod22.z.record(import_zod22.z.string(), import_zod22.z.unknown()).optional(),
-  llmTraceSteps: import_zod22.z.array(LLMTraceStepSchema).optional()
-});
-var EvalRunResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  targetId: import_zod22.z.string(),
-  targetName: import_zod22.z.string().optional(),
-  scenarioId: import_zod22.z.string(),
-  scenarioName: import_zod22.z.string(),
-  modelConfig: ModelConfigSchema.optional(),
-  assertionResults: import_zod22.z.array(AssertionResultSchema),
-  metrics: EvalMetricsSchema.optional(),
-  passed: import_zod22.z.number(),
-  failed: import_zod22.z.number(),
-  passRate: import_zod22.z.number(),
-  duration: import_zod22.z.number(),
-  outputText: import_zod22.z.string().optional(),
-  files: import_zod22.z.array(ExpectedFileSchema).optional(),
-  startedAt: import_zod22.z.string().optional(),
-  completedAt: import_zod22.z.string().optional(),
-  llmTrace: LLMTraceSchema.optional()
-});
-var PromptResultSchema = import_zod22.z.object({
-  text: import_zod22.z.string(),
-  files: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  finishReason: import_zod22.z.string().optional(),
-  reasoning: import_zod22.z.string().optional(),
-  reasoningDetails: import_zod22.z.unknown().optional(),
-  toolCalls: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  toolResults: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  warnings: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  sources: import_zod22.z.array(import_zod22.z.unknown()).optional(),
-  steps: import_zod22.z.array(import_zod22.z.unknown()),
-  generationTimeMs: import_zod22.z.number(),
-  prompt: import_zod22.z.string(),
-  systemPrompt: import_zod22.z.string(),
-  usage: import_zod22.z.object({
-    totalTokens: import_zod22.z.number().optional(),
-    totalMicrocentsSpent: import_zod22.z.number().optional()
-  })
-});
-var EvaluationResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  runId: import_zod22.z.string(),
-  timestamp: import_zod22.z.number(),
-  promptResult: PromptResultSchema,
-  testResults: import_zod22.z.array(import_zod22.z.unknown()),
-  tags: import_zod22.z.array(import_zod22.z.string()).optional(),
-  feedback: import_zod22.z.string().optional(),
-  score: import_zod22.z.number(),
-  suiteId: import_zod22.z.string().optional()
-});
-var LeanEvaluationResultSchema = import_zod22.z.object({
-  id: import_zod22.z.string(),
-  runId: import_zod22.z.string(),
-  timestamp: import_zod22.z.number(),
-  tags: import_zod22.z.array(import_zod22.z.string()).optional(),
-  scenarioId: import_zod22.z.string(),
-  scenarioVersion: import_zod22.z.number().optional(),
-  targetId: import_zod22.z.string(),
-  targetVersion: import_zod22.z.number().optional(),
-  suiteId: import_zod22.z.string().optional(),
-  score: import_zod22.z.number(),
-  time: import_zod22.z.number().optional(),
-  microcentsSpent: import_zod22.z.number().optional()
-});
+var import_zod25 = require("zod");
 
 // src/evaluation/eval-run.ts
 var import_zod24 = require("zod");
@@ -930,11 +886,97 @@ var EvaluationLogSchema = import_zod24.z.object({
 });
 var LLM_TIMEOUT = 12e4;
 
+// src/evaluation/eval-result.ts
+var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
+  AssertionResultStatus2["PASSED"] = "passed";
+  AssertionResultStatus2["FAILED"] = "failed";
+  AssertionResultStatus2["SKIPPED"] = "skipped";
+  AssertionResultStatus2["ERROR"] = "error";
+  return AssertionResultStatus2;
+})(AssertionResultStatus || {});
+var AssertionResultSchema = import_zod25.z.object({
+  id: import_zod25.z.string(),
+  assertionId: import_zod25.z.string(),
+  assertionType: import_zod25.z.string(),
+  assertionName: import_zod25.z.string(),
+  status: import_zod25.z.enum(AssertionResultStatus),
+  message: import_zod25.z.string().optional(),
+  expected: import_zod25.z.string().optional(),
+  actual: import_zod25.z.string().optional(),
+  duration: import_zod25.z.number().optional(),
+  details: import_zod25.z.record(import_zod25.z.string(), import_zod25.z.unknown()).optional(),
+  llmTraceSteps: import_zod25.z.array(LLMTraceStepSchema).optional()
+});
+var EvalRunResultSchema = import_zod25.z.object({
+  id: import_zod25.z.string(),
+  targetId: import_zod25.z.string(),
+  targetName: import_zod25.z.string().optional(),
+  scenarioId: import_zod25.z.string(),
+  scenarioName: import_zod25.z.string(),
+  modelConfig: ModelConfigSchema.optional(),
+  assertionResults: import_zod25.z.array(AssertionResultSchema),
+  metrics: EvalMetricsSchema.optional(),
+  passed: import_zod25.z.number(),
+  failed: import_zod25.z.number(),
+  passRate: import_zod25.z.number(),
+  duration: import_zod25.z.number(),
+  outputText: import_zod25.z.string().optional(),
+  files: import_zod25.z.array(ExpectedFileSchema).optional(),
+  fileDiffs: import_zod25.z.array(DiffContentSchema).optional(),
+  startedAt: import_zod25.z.string().optional(),
+  completedAt: import_zod25.z.string().optional(),
+  llmTrace: LLMTraceSchema.optional()
+});
+var PromptResultSchema = import_zod25.z.object({
+  text: import_zod25.z.string(),
+  files: import_zod25.z.array(import_zod25.z.unknown()).optional(),
+  finishReason: import_zod25.z.string().optional(),
+  reasoning: import_zod25.z.string().optional(),
+  reasoningDetails: import_zod25.z.unknown().optional(),
+  toolCalls: import_zod25.z.array(import_zod25.z.unknown()).optional(),
+  toolResults: import_zod25.z.array(import_zod25.z.unknown()).optional(),
+  warnings: import_zod25.z.array(import_zod25.z.unknown()).optional(),
+  sources: import_zod25.z.array(import_zod25.z.unknown()).optional(),
+  steps: import_zod25.z.array(import_zod25.z.unknown()),
+  generationTimeMs: import_zod25.z.number(),
+  prompt: import_zod25.z.string(),
+  systemPrompt: import_zod25.z.string(),
+  usage: import_zod25.z.object({
+    totalTokens: import_zod25.z.number().optional(),
+    totalMicrocentsSpent: import_zod25.z.number().optional()
+  })
+});
+var EvaluationResultSchema = import_zod25.z.object({
+  id: import_zod25.z.string(),
+  runId: import_zod25.z.string(),
+  timestamp: import_zod25.z.number(),
+  promptResult: PromptResultSchema,
+  testResults: import_zod25.z.array(import_zod25.z.unknown()),
+  tags: import_zod25.z.array(import_zod25.z.string()).optional(),
+  feedback: import_zod25.z.string().optional(),
+  score: import_zod25.z.number(),
+  suiteId: import_zod25.z.string().optional()
+});
+var LeanEvaluationResultSchema = import_zod25.z.object({
+  id: import_zod25.z.string(),
+  runId: import_zod25.z.string(),
+  timestamp: import_zod25.z.number(),
+  tags: import_zod25.z.array(import_zod25.z.string()).optional(),
+  scenarioId: import_zod25.z.string(),
+  scenarioVersion: import_zod25.z.number().optional(),
+  targetId: import_zod25.z.string(),
+  targetVersion: import_zod25.z.number().optional(),
+  suiteId: import_zod25.z.string().optional(),
+  score: import_zod25.z.number(),
+  time: import_zod25.z.number().optional(),
+  microcentsSpent: import_zod25.z.number().optional()
+});
+
 // src/project/project.ts
-var import_zod25 = require("zod");
+var import_zod26 = require("zod");
 var ProjectSchema = BaseEntitySchema.extend({
-  appId: import_zod25.z.string().optional().describe("The ID of the app in Dev Center"),
-  appSecret: import_zod25.z.string().optional().describe("The secret of the app in Dev Center")
+  appId: import_zod26.z.string().optional().describe("The ID of the app in Dev Center"),
+  appSecret: import_zod26.z.string().optional().describe("The secret of the app in Dev Center")
 });
 var CreateProjectInputSchema = ProjectSchema.omit({
   id: true,
@@ -945,10 +987,10 @@ var CreateProjectInputSchema = ProjectSchema.omit({
 var UpdateProjectInputSchema = CreateProjectInputSchema.partial();
 
 // src/template/template.ts
-var import_zod26 = require("zod");
+var import_zod27 = require("zod");
 var TemplateSchema = TenantEntitySchema.extend({
   /** URL to download the template from */
-  downloadUrl: import_zod26.z.url()
+  downloadUrl: import_zod27.z.url()
 });
 var CreateTemplateInputSchema = TemplateSchema.omit({
   id: true,
@@ -966,9 +1008,11 @@ var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
   ApiCallSchema,
   AssertionResultSchema,
   AssertionResultStatus,
+  AssertionSchema,
   BaseEntitySchema,
   BaseTestSchema,
   BuildCheckTestSchema,
+  BuildPassedAssertionSchema,
   CommandExecutionSchema,
   CommandExecutionTestSchema,
   CreateAgentInputSchema,
@@ -1010,6 +1054,7 @@ var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
   LeanEvaluationResultSchema,
   LiveTraceEventSchema,
   LiveTraceEventType,
+  LlmJudgeAssertionSchema,
   LocalProjectConfigSchema,
   MCPServerConfigSchema,
   MetaSiteConfigSchema,
@@ -1025,6 +1070,7 @@ var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
   SkillMetadataSchema,
   SkillSchema,
   SkillVersionSchema,
+  SkillWasCalledAssertionSchema,
   SkillsGroupSchema,
   TRACE_EVENT_PREFIX,
   TargetSchema,