@wix/evalforge-types 0.3.0

package/build/index.mjs

@@ -0,0 +1,928 @@
+// src/common/base-entity.ts
+import { z } from "zod";
+var BaseEntitySchema = z.object({
+  id: z.string(),
+  name: z.string().min(1),
+  description: z.string(),
+  createdAt: z.string(),
+  updatedAt: z.string(),
+  deleted: z.boolean().optional()
+});
+var TenantEntitySchema = BaseEntitySchema.extend({
+  projectId: z.string()
+});
+
+// src/common/mcp.ts
+import { z as z2 } from "zod";
+var MCPServerConfigSchema = z2.object({
+  /** Unique name for this MCP server */
+  name: z2.string(),
+  /** Command to start the MCP server */
+  command: z2.string(),
+  /** Command line arguments */
+  args: z2.array(z2.string()).optional(),
+  /** Environment variables for the server process */
+  envVars: z2.record(z2.string(), z2.string()).optional(),
+  /** Tools to disable for this MCP server */
+  disabledTools: z2.array(z2.string()).optional()
+});
+
+// src/common/models.ts
+import { z as z3 } from "zod";
+var ModelIds = /* @__PURE__ */ ((ModelIds2) => {
+  ModelIds2["CLAUDE_3_HAIKU_1_0"] = "CLAUDE_3_HAIKU_1_0";
+  ModelIds2["CLAUDE_3_OPUS_1_0"] = "CLAUDE_3_OPUS_1_0";
+  ModelIds2["CLAUDE_3_SONNET_1_0"] = "CLAUDE_3_SONNET_1_0";
+  ModelIds2["CLAUDE_3_5_SONNET_1_0"] = "CLAUDE_3_5_SONNET_1_0";
+  ModelIds2["CLAUDE_3_5_SONNET_2_0"] = "CLAUDE_3_5_SONNET_2_0";
+  ModelIds2["CLAUDE_3_7_SONNET_1_0"] = "CLAUDE_3_7_SONNET_1_0";
+  ModelIds2["CLAUDE_4_OPUS_1_0"] = "CLAUDE_4_OPUS_1_0";
+  ModelIds2["CLAUDE_4_SONNET_1_0"] = "CLAUDE_4_SONNET_1_0";
+  return ModelIds2;
+})(ModelIds || {});
+var ModelIdsSchema = z3.enum(ModelIds);
+var ModelConfigSchema = z3.object({
+  model: ModelIdsSchema,
+  temperature: z3.number().min(0).max(1).optional(),
+  maxTokens: z3.number().min(1).optional()
+});
+var ModelPricingSchema = z3.object({
+  inputPer1M: z3.number(),
+  outputPer1M: z3.number()
+});
+var ModelSchema = z3.object({
+  /** AI Gateway model ID */
+  id: ModelIdsSchema,
+  /** Display name */
+  name: z3.string(),
+  /** Provider (always 'anthropic') */
+  provider: z3.literal("anthropic"),
+  /** Provider's model identifier (e.g., "claude-3-5-sonnet-20241022") */
+  providerModelId: z3.string(),
+  /** Pricing per 1M tokens */
+  pricing: ModelPricingSchema
+});
+var AVAILABLE_MODELS = [
+  {
+    id: "CLAUDE_4_SONNET_1_0" /* CLAUDE_4_SONNET_1_0 */,
+    name: "Claude 4 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-4-sonnet",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_4_OPUS_1_0" /* CLAUDE_4_OPUS_1_0 */,
+    name: "Claude 4 Opus",
+    provider: "anthropic",
+    providerModelId: "claude-4-opus",
+    pricing: { inputPer1M: 15, outputPer1M: 75 }
+  },
+  {
+    id: "CLAUDE_3_7_SONNET_1_0" /* CLAUDE_3_7_SONNET_1_0 */,
+    name: "Claude 3.7 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-3-7-sonnet",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_5_SONNET_2_0" /* CLAUDE_3_5_SONNET_2_0 */,
+    name: "Claude 3.5 Sonnet v2",
+    provider: "anthropic",
+    providerModelId: "claude-3-5-sonnet-20241022",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_5_SONNET_1_0" /* CLAUDE_3_5_SONNET_1_0 */,
+    name: "Claude 3.5 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-3-5-sonnet-20240620",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_OPUS_1_0" /* CLAUDE_3_OPUS_1_0 */,
+    name: "Claude 3 Opus",
+    provider: "anthropic",
+    providerModelId: "claude-3-opus-20240229",
+    pricing: { inputPer1M: 15, outputPer1M: 75 }
+  },
+  {
+    id: "CLAUDE_3_SONNET_1_0" /* CLAUDE_3_SONNET_1_0 */,
+    name: "Claude 3 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-3-sonnet-20240229",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_HAIKU_1_0" /* CLAUDE_3_HAIKU_1_0 */,
+    name: "Claude 3 Haiku",
+    provider: "anthropic",
+    providerModelId: "claude-3-haiku-20240307",
+    pricing: { inputPer1M: 0.25, outputPer1M: 1.25 }
+  }
+];
+var AVAILABLE_MODELS_MAP = Object.fromEntries(
+  AVAILABLE_MODELS.map((model) => [model.id, model])
+);
+
+// src/target/target.ts
+var TargetSchema = TenantEntitySchema.extend({
+  // Base for all testable entities
+  // Specific targets add their own fields
+});
+
+// src/target/agent.ts
+import { z as z4 } from "zod";
+var AgentSchema = TargetSchema.extend({
+  /** Command to run the agent */
+  runCommand: z4.string(),
+  /** Optional model configuration override */
+  modelConfig: ModelConfigSchema.optional()
+});
+var CreateAgentInputSchema = AgentSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateAgentInputSchema = CreateAgentInputSchema.partial();
+
+// src/target/skill.ts
+import { z as z5 } from "zod";
+var SkillMetadataSchema = z5.object({
+  name: z5.string(),
+  description: z5.string(),
+  allowedTools: z5.array(z5.string()).optional(),
+  skills: z5.array(z5.string()).optional()
+});
+var SkillVersionSchema = z5.object({
+  id: z5.string(),
+  skillId: z5.string(),
+  skillMd: z5.string(),
+  metadata: SkillMetadataSchema,
+  model: ModelConfigSchema.optional(),
+  systemPrompt: z5.string().optional(),
+  version: z5.number(),
+  createdAt: z5.string(),
+  notes: z5.string().optional()
+});
+var SkillSchema = TargetSchema.extend({
+  /** The current SKILL.md content */
+  skillMd: z5.string()
+});
+var CreateSkillInputSchema = SkillSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateSkillInputSchema = CreateSkillInputSchema.partial();
+
+// src/target/skills-group.ts
+import { z as z6 } from "zod";
+var SkillsGroupSchema = TenantEntitySchema.extend({
+  /** IDs of skills in this group */
+  skillIds: z6.array(z6.string())
+});
+var CreateSkillsGroupInputSchema = SkillsGroupSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateSkillsGroupInputSchema = CreateSkillsGroupInputSchema.partial();
+
+// src/test/index.ts
+import { z as z17 } from "zod";
+
+// src/test/base.ts
+import { z as z7 } from "zod";
+var TestType = /* @__PURE__ */ ((TestType2) => {
+  TestType2["LLM"] = "LLM";
+  TestType2["TOOL"] = "TOOL";
+  TestType2["SITE_CONFIG"] = "SITE_CONFIG";
+  TestType2["COMMAND_EXECUTION"] = "COMMAND_EXECUTION";
+  TestType2["FILE_PRESENCE"] = "FILE_PRESENCE";
+  TestType2["FILE_CONTENT"] = "FILE_CONTENT";
+  TestType2["BUILD_CHECK"] = "BUILD_CHECK";
+  TestType2["VITEST"] = "VITEST";
+  TestType2["PLAYWRIGHT_NL"] = "PLAYWRIGHT_NL";
+  return TestType2;
+})(TestType || {});
+var TestTypeSchema = z7.enum(TestType);
+var TestImportance = /* @__PURE__ */ ((TestImportance2) => {
+  TestImportance2["LOW"] = "low";
+  TestImportance2["MEDIUM"] = "medium";
+  TestImportance2["HIGH"] = "high";
+  TestImportance2["CRITICAL"] = "critical";
+  return TestImportance2;
+})(TestImportance || {});
+var TestImportanceSchema = z7.enum(TestImportance);
+var BaseTestSchema = z7.object({
+  id: z7.string(),
+  type: TestTypeSchema,
+  name: z7.string().min(3),
+  description: z7.string().optional(),
+  importance: TestImportanceSchema.optional()
+});
+
+// src/test/llm.ts
+import { z as z8 } from "zod";
+var LLMTestSchema = BaseTestSchema.extend({
+  type: z8.literal("LLM" /* LLM */),
+  /** Maximum steps for the LLM to take */
+  maxSteps: z8.number().min(1).max(100),
+  /** Prompt to send to the evaluator */
+  prompt: z8.string().min(1),
+  /** ID of the evaluator agent to use */
+  evaluatorId: z8.string()
+});
+
+// src/test/tool.ts
+import { z as z9 } from "zod";
+var ToolTestSchema = BaseTestSchema.extend({
+  type: z9.literal("TOOL" /* TOOL */),
+  /** Name of the tool that should be called */
+  toolName: z9.string().min(3),
+  /** Expected arguments for the tool call */
+  args: z9.record(z9.string(), z9.any()),
+  /** Expected content in the tool results */
+  resultsContent: z9.string()
+});
+
+// src/test/site-config.ts
+import { z as z10 } from "zod";
+var SiteConfigTestSchema = BaseTestSchema.extend({
+  type: z10.literal("SITE_CONFIG" /* SITE_CONFIG */),
+  /** URL to call */
+  url: z10.string().url(),
+  /** HTTP method */
+  method: z10.enum(["GET", "POST"]),
+  /** Request body (for POST) */
+  body: z10.string().optional(),
+  /** Expected HTTP status code */
+  expectedStatusCode: z10.number().int().min(100).max(599),
+  /** Expected response content */
+  expectedResponse: z10.string().optional(),
+  /** JMESPath expression to extract from response */
+  expectedResponseJMESPath: z10.string().optional()
+});
+
+// src/test/command-execution.ts
+import { z as z11 } from "zod";
+var AllowedCommands = [
+  "yarn install --no-immutable && yarn build",
+  "npm run build",
+  "yarn typecheck"
+];
+var CommandExecutionTestSchema = BaseTestSchema.extend({
+  type: z11.literal("COMMAND_EXECUTION" /* COMMAND_EXECUTION */),
+  /** Command to execute (must be in AllowedCommands) */
+  command: z11.string().refine((value) => AllowedCommands.includes(value), {
+    message: `Command must be one of: ${AllowedCommands.join(", ")}`
+  }),
+  /** Expected exit code (default: 0) */
+  expectedExitCode: z11.number().default(0).optional()
+});
+
+// src/test/file-presence.ts
+import { z as z12 } from "zod";
+var FilePresenceTestSchema = BaseTestSchema.extend({
+  type: z12.literal("FILE_PRESENCE" /* FILE_PRESENCE */),
+  /** Paths to check */
+  paths: z12.array(z12.string()),
+  /** Whether files should exist (true) or not exist (false) */
+  shouldExist: z12.boolean()
+});
+
+// src/test/file-content.ts
+import { z as z13 } from "zod";
+var FileContentCheckSchema = z13.object({
+  /** Strings that must be present in the file */
+  contains: z13.array(z13.string()).optional(),
+  /** Strings that must NOT be present in the file */
+  notContains: z13.array(z13.string()).optional(),
+  /** Regex pattern the content must match */
+  matches: z13.string().optional(),
+  /** JSON path checks for structured content */
+  jsonPath: z13.array(
+    z13.object({
+      path: z13.string(),
+      value: z13.unknown()
+    })
+  ).optional(),
+  /** Lines that should be added (for diff checking) */
+  added: z13.array(z13.string()).optional(),
+  /** Lines that should be removed (for diff checking) */
+  removed: z13.array(z13.string()).optional()
+});
+var FileContentTestSchema = BaseTestSchema.extend({
+  type: z13.literal("FILE_CONTENT" /* FILE_CONTENT */),
+  /** Path to the file to check */
+  path: z13.string(),
+  /** Content checks to perform */
+  checks: FileContentCheckSchema
+});
+
+// src/test/build-check.ts
+import { z as z14 } from "zod";
+var BuildCheckTestSchema = BaseTestSchema.extend({
+  type: z14.literal("BUILD_CHECK" /* BUILD_CHECK */),
+  /** Build command to execute */
+  command: z14.string(),
+  /** Whether the build should succeed */
+  expectSuccess: z14.boolean(),
+  /** Maximum allowed warnings (optional) */
+  allowedWarnings: z14.number().optional(),
+  /** Timeout in milliseconds */
+  timeout: z14.number().optional()
+});
+
+// src/test/vitest.ts
+import { z as z15 } from "zod";
+var VitestTestSchema = BaseTestSchema.extend({
+  type: z15.literal("VITEST" /* VITEST */),
+  /** Test file content */
+  testFile: z15.string(),
+  /** Name of the test file */
+  testFileName: z15.string(),
+  /** Minimum pass rate required (0-100) */
+  minPassRate: z15.number().min(0).max(100)
+});
+
+// src/test/playwright-nl.ts
+import { z as z16 } from "zod";
+var PlaywrightNLTestSchema = BaseTestSchema.extend({
+  type: z16.literal("PLAYWRIGHT_NL" /* PLAYWRIGHT_NL */),
+  /** Natural language steps to execute */
+  steps: z16.array(z16.string()),
+  /** Expected outcome description */
+  expectedOutcome: z16.string(),
+  /** Timeout in milliseconds */
+  timeout: z16.number().optional()
+});
+
+// src/test/index.ts
+var TestSchema = z17.discriminatedUnion("type", [
+  LLMTestSchema,
+  ToolTestSchema,
+  SiteConfigTestSchema,
+  CommandExecutionTestSchema,
+  FilePresenceTestSchema,
+  FileContentTestSchema,
+  BuildCheckTestSchema,
+  VitestTestSchema,
+  PlaywrightNLTestSchema
+]);
+
+// src/scenario/environment.ts
+import { z as z18 } from "zod";
+var LocalProjectConfigSchema = z18.object({
+  /** Template ID to use for the local project */
+  templateId: z18.string().optional(),
+  /** Files to create in the project */
+  files: z18.array(
+    z18.object({
+      path: z18.string().min(1),
+      content: z18.string().min(1)
+    })
+  ).optional()
+});
+var MetaSiteConfigSchema = z18.object({
+  configurations: z18.array(
+    z18.object({
+      name: z18.string().min(1),
+      apiCalls: z18.array(
+        z18.object({
+          url: z18.string().url(),
+          method: z18.enum(["POST", "PUT"]),
+          body: z18.string()
+        })
+      )
+    })
+  ).optional()
+});
+var EnvironmentSchema = z18.object({
+  /** Local project configuration */
+  localProject: LocalProjectConfigSchema.optional(),
+  /** Meta site configuration */
+  metaSite: MetaSiteConfigSchema.optional()
+});
+
+// src/scenario/test-scenario.ts
+import { z as z19 } from "zod";
+var ExpectedFileSchema = z19.object({
+  /** Relative path where the file should be created */
+  path: z19.string(),
+  /** Optional expected content */
+  content: z19.string().optional()
+});
+var TestScenarioSchema = TenantEntitySchema.extend({
+  /** The prompt sent to the agent to trigger the task */
+  triggerPrompt: z19.string().min(10),
+  /** ID of the template to use for this scenario */
+  templateId: z19.string().optional()
+});
+var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateTestScenarioInputSchema = CreateTestScenarioInputSchema.partial();
+
+// src/suite/test-suite.ts
+import { z as z20 } from "zod";
+var TestSuiteSchema = TenantEntitySchema.extend({
+  /** IDs of test scenarios in this suite */
+  scenarioIds: z20.array(z20.string())
+});
+var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateTestSuiteInputSchema = CreateTestSuiteInputSchema.partial();
+
+// src/evaluation/metrics.ts
+import { z as z21 } from "zod";
+var TokenUsageSchema = z21.object({
+  prompt: z21.number(),
+  completion: z21.number(),
+  total: z21.number()
+});
+var EvalMetricsSchema = z21.object({
+  totalAssertions: z21.number(),
+  passed: z21.number(),
+  failed: z21.number(),
+  skipped: z21.number(),
+  errors: z21.number(),
+  passRate: z21.number(),
+  avgDuration: z21.number(),
+  totalDuration: z21.number()
+});
+var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
+  EvalStatus2["PENDING"] = "pending";
+  EvalStatus2["RUNNING"] = "running";
+  EvalStatus2["COMPLETED"] = "completed";
+  EvalStatus2["FAILED"] = "failed";
+  EvalStatus2["CANCELLED"] = "cancelled";
+  return EvalStatus2;
+})(EvalStatus || {});
+var EvalStatusSchema = z21.enum(EvalStatus);
+var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
+  LLMStepType2["COMPLETION"] = "completion";
+  LLMStepType2["TOOL_USE"] = "tool_use";
+  LLMStepType2["TOOL_RESULT"] = "tool_result";
+  LLMStepType2["THINKING"] = "thinking";
+  return LLMStepType2;
+})(LLMStepType || {});
+var LLMTraceStepSchema = z21.object({
+  id: z21.string(),
+  stepNumber: z21.number(),
+  type: z21.enum(LLMStepType),
+  model: z21.string(),
+  provider: z21.string(),
+  startedAt: z21.string(),
+  durationMs: z21.number(),
+  tokenUsage: TokenUsageSchema,
+  costUsd: z21.number(),
+  toolName: z21.string().optional(),
+  toolArguments: z21.string().optional(),
+  inputPreview: z21.string().optional(),
+  outputPreview: z21.string().optional(),
+  success: z21.boolean(),
+  error: z21.string().optional()
+});
+var LLMBreakdownStatsSchema = z21.object({
+  count: z21.number(),
+  durationMs: z21.number(),
+  tokens: z21.number(),
+  costUsd: z21.number()
+});
+var LLMTraceSummarySchema = z21.object({
+  totalSteps: z21.number(),
+  totalDurationMs: z21.number(),
+  totalTokens: TokenUsageSchema,
+  totalCostUsd: z21.number(),
+  stepTypeBreakdown: z21.record(z21.string(), LLMBreakdownStatsSchema).optional(),
+  modelBreakdown: z21.record(z21.string(), LLMBreakdownStatsSchema),
+  modelsUsed: z21.array(z21.string())
+});
+var LLMTraceSchema = z21.object({
+  id: z21.string(),
+  steps: z21.array(LLMTraceStepSchema),
+  summary: LLMTraceSummarySchema
+});
+
+// src/evaluation/eval-result.ts
+import { z as z22 } from "zod";
+var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
+  AssertionResultStatus2["PASSED"] = "passed";
+  AssertionResultStatus2["FAILED"] = "failed";
+  AssertionResultStatus2["SKIPPED"] = "skipped";
+  AssertionResultStatus2["ERROR"] = "error";
+  return AssertionResultStatus2;
+})(AssertionResultStatus || {});
+var AssertionResultSchema = z22.object({
+  id: z22.string(),
+  assertionId: z22.string(),
+  assertionType: z22.string(),
+  assertionName: z22.string(),
+  status: z22.enum(AssertionResultStatus),
+  message: z22.string().optional(),
+  expected: z22.string().optional(),
+  actual: z22.string().optional(),
+  duration: z22.number().optional(),
+  details: z22.record(z22.string(), z22.unknown()).optional(),
+  llmTraceSteps: z22.array(LLMTraceStepSchema).optional()
+});
+var EvalRunResultSchema = z22.object({
+  id: z22.string(),
+  targetId: z22.string(),
+  targetName: z22.string().optional(),
+  scenarioId: z22.string(),
+  scenarioName: z22.string(),
+  modelConfig: ModelConfigSchema.optional(),
+  assertionResults: z22.array(AssertionResultSchema),
+  metrics: EvalMetricsSchema.optional(),
+  passed: z22.number(),
+  failed: z22.number(),
+  passRate: z22.number(),
+  duration: z22.number(),
+  outputText: z22.string().optional(),
+  files: z22.array(ExpectedFileSchema).optional(),
+  startedAt: z22.string().optional(),
+  completedAt: z22.string().optional(),
+  llmTrace: LLMTraceSchema.optional()
+});
+var PromptResultSchema = z22.object({
+  text: z22.string(),
+  files: z22.array(z22.unknown()).optional(),
+  finishReason: z22.string().optional(),
+  reasoning: z22.string().optional(),
+  reasoningDetails: z22.unknown().optional(),
+  toolCalls: z22.array(z22.unknown()).optional(),
+  toolResults: z22.array(z22.unknown()).optional(),
+  warnings: z22.array(z22.unknown()).optional(),
+  sources: z22.array(z22.unknown()).optional(),
+  steps: z22.array(z22.unknown()),
+  generationTimeMs: z22.number(),
+  prompt: z22.string(),
+  systemPrompt: z22.string(),
+  usage: z22.object({
+    totalTokens: z22.number().optional(),
+    totalMicrocentsSpent: z22.number().optional()
+  })
+});
+var EvaluationResultSchema = z22.object({
+  id: z22.string(),
+  runId: z22.string(),
+  timestamp: z22.number(),
+  promptResult: PromptResultSchema,
+  testResults: z22.array(z22.unknown()),
+  tags: z22.array(z22.string()).optional(),
+  feedback: z22.string().optional(),
+  score: z22.number(),
+  suiteId: z22.string().optional()
+});
+var LeanEvaluationResultSchema = z22.object({
+  id: z22.string(),
+  runId: z22.string(),
+  timestamp: z22.number(),
+  tags: z22.array(z22.string()).optional(),
+  scenarioId: z22.string(),
+  scenarioVersion: z22.number().optional(),
+  targetId: z22.string(),
+  targetVersion: z22.number().optional(),
+  suiteId: z22.string().optional(),
+  score: z22.number(),
+  time: z22.number().optional(),
+  microcentsSpent: z22.number().optional()
+});
+
+// src/evaluation/eval-run.ts
+import { z as z24 } from "zod";
+
+// src/evaluation/live-trace.ts
+import { z as z23 } from "zod";
+var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
+  LiveTraceEventType2["THINKING"] = "thinking";
+  LiveTraceEventType2["TOOL_USE"] = "tool_use";
+  LiveTraceEventType2["COMPLETION"] = "completion";
+  LiveTraceEventType2["TOOL_RESULT"] = "tool_result";
+  return LiveTraceEventType2;
+})(LiveTraceEventType || {});
+var LiveTraceEventSchema = z23.object({
+  /** The evaluation run ID */
+  evalRunId: z23.string(),
+  /** The scenario ID being executed */
+  scenarioId: z23.string(),
+  /** The scenario name for display */
+  scenarioName: z23.string(),
+  /** The target ID (skill, agent, etc.) */
+  targetId: z23.string(),
+  /** The target name for display */
+  targetName: z23.string(),
+  /** Step number in the current scenario execution */
+  stepNumber: z23.number(),
+  /** Type of trace event */
+  type: z23.enum(LiveTraceEventType),
+  /** Tool name if this is a tool_use event */
+  toolName: z23.string().optional(),
+  /** Tool arguments preview (truncated JSON) */
+  toolArgs: z23.string().optional(),
+  /** Output preview (truncated text) */
+  outputPreview: z23.string().optional(),
+  /** Timestamp when this event occurred */
+  timestamp: z23.string(),
+  /** Whether this is the final event for this scenario */
+  isComplete: z23.boolean()
+});
+var TRACE_EVENT_PREFIX = "TRACE_EVENT:";
+function parseTraceEventLine(line) {
+  if (!line.startsWith(TRACE_EVENT_PREFIX)) {
+    return null;
+  }
+  try {
+    const jsonStr = line.slice(TRACE_EVENT_PREFIX.length);
+    const parsed = JSON.parse(jsonStr);
+    const result = LiveTraceEventSchema.safeParse(parsed);
+    return result.success ? result.data : null;
+  } catch {
+    return null;
+  }
+}
+function formatTraceEventLine(event) {
+  return `${TRACE_EVENT_PREFIX}${JSON.stringify(event)}`;
+}
+
+// src/evaluation/eval-run.ts
+var TriggerType = /* @__PURE__ */ ((TriggerType2) => {
+  TriggerType2["RESOURCES_UPDATED"] = "RESOURCES_UPDATED";
+  TriggerType2["MCP_VERSION_RELEASE"] = "MCP_VERSION_RELEASE";
+  TriggerType2["MCP_PREVIEW_CREATED"] = "MCP_PREVIEW_CREATED";
+  TriggerType2["MANUAL"] = "MANUAL";
+  return TriggerType2;
+})(TriggerType || {});
+var TriggerMetadataSchema = z24.object({
+  version: z24.string().optional(),
+  resourceUpdated: z24.array(z24.string()).optional()
+});
+var TriggerSchema = z24.object({
+  id: z24.string(),
+  metadata: TriggerMetadataSchema.optional(),
+  type: z24.enum(TriggerType)
+});
+var FailureCategory = /* @__PURE__ */ ((FailureCategory2) => {
+  FailureCategory2["MISSING_FILE"] = "missing_file";
+  FailureCategory2["WRONG_CONTENT"] = "wrong_content";
+  FailureCategory2["BUILD_ERROR"] = "build_error";
+  FailureCategory2["TEST_FAILURE"] = "test_failure";
+  FailureCategory2["RUNTIME_ERROR"] = "runtime_error";
+  FailureCategory2["PERFORMANCE"] = "performance";
+  return FailureCategory2;
+})(FailureCategory || {});
+var FailureSeverity = /* @__PURE__ */ ((FailureSeverity2) => {
+  FailureSeverity2["CRITICAL"] = "critical";
+  FailureSeverity2["HIGH"] = "high";
+  FailureSeverity2["MEDIUM"] = "medium";
+  FailureSeverity2["LOW"] = "low";
+  return FailureSeverity2;
+})(FailureSeverity || {});
+var DiffLineTypeSchema = z24.enum(["added", "removed", "unchanged"]);
+var DiffLineSchema = z24.object({
+  type: DiffLineTypeSchema,
+  content: z24.string(),
+  lineNumber: z24.number()
+});
+var DiffContentSchema = z24.object({
+  path: z24.string(),
+  expected: z24.string(),
+  actual: z24.string(),
+  diffLines: z24.array(DiffLineSchema)
+});
+var CommandExecutionSchema = z24.object({
+  command: z24.string(),
+  exitCode: z24.number(),
+  output: z24.string().optional(),
+  duration: z24.number()
+});
+var FileModificationSchema = z24.object({
+  path: z24.string(),
+  action: z24.enum(["created", "modified", "deleted"])
+});
+var ApiCallSchema = z24.object({
+  endpoint: z24.string(),
+  tokensUsed: z24.number(),
+  duration: z24.number()
+});
+var ExecutionTraceSchema = z24.object({
+  commands: z24.array(CommandExecutionSchema),
+  filesModified: z24.array(FileModificationSchema),
+  apiCalls: z24.array(ApiCallSchema),
+  totalDuration: z24.number()
+});
+var FailureAnalysisSchema = z24.object({
+  category: z24.enum(FailureCategory),
+  severity: z24.enum(FailureSeverity),
+  summary: z24.string(),
+  details: z24.string(),
+  rootCause: z24.string(),
+  suggestedFix: z24.string(),
+  relatedAssertions: z24.array(z24.string()),
+  codeSnippet: z24.string().optional(),
+  similarIssues: z24.array(z24.string()).optional(),
+  patternId: z24.string().optional(),
+  // Extended fields for detailed debugging
+  diff: DiffContentSchema.optional(),
+  executionTrace: ExecutionTraceSchema.optional()
+});
+var EvalRunSchema = TenantEntitySchema.extend({
+  /** Agent ID for this run */
+  agentId: z24.string().optional(),
+  /** Skills group ID for this run */
+  skillsGroupId: z24.string().optional(),
+  /** Scenario IDs to run */
+  scenarioIds: z24.array(z24.string()),
+  /** Current status */
+  status: EvalStatusSchema,
+  /** Progress percentage (0-100) */
+  progress: z24.number(),
+  /** Results for each scenario/target combination */
+  results: z24.array(EvalRunResultSchema),
+  /** Aggregated metrics across all results */
+  aggregateMetrics: EvalMetricsSchema,
+  /** Failure analyses */
+  failureAnalyses: z24.array(FailureAnalysisSchema).optional(),
+  /** Aggregated LLM trace summary */
+  llmTraceSummary: LLMTraceSummarySchema.optional(),
+  /** What triggered this run */
+  trigger: TriggerSchema.optional(),
+  /** When the run started (set when evaluation is triggered) */
+  startedAt: z24.string().optional(),
+  /** When the run completed */
+  completedAt: z24.string().optional(),
+  /** Live trace events captured during execution (for playback on results page) */
+  liveTraceEvents: z24.array(LiveTraceEventSchema).optional()
+});
+var CreateEvalRunInputSchema = EvalRunSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  status: true,
+  progress: true,
+  results: true,
+  aggregateMetrics: true,
+  startedAt: true,
+  completedAt: true
+});
+var EvaluationProgressSchema = z24.object({
+  runId: z24.string(),
+  targetId: z24.string(),
+  totalScenarios: z24.number(),
+  completedScenarios: z24.number(),
+  scenarioProgress: z24.array(
+    z24.object({
+      scenarioId: z24.string(),
+      currentStep: z24.string(),
+      error: z24.string().optional()
+    })
+  ),
+  createdAt: z24.number()
+});
+var EvaluationLogSchema = z24.object({
+  runId: z24.string(),
+  scenarioId: z24.string(),
+  log: z24.object({
+    level: z24.enum(["info", "error", "debug"]),
+    message: z24.string().optional(),
+    args: z24.array(z24.any()).optional(),
+    error: z24.string().optional()
+  })
+});
+var LLM_TIMEOUT = 12e4;
+
+// src/project/project.ts
+import { z as z25 } from "zod";
+var ProjectSchema = BaseEntitySchema.extend({
+  appId: z25.string().optional().describe("The ID of the app in Dev Center"),
+  appSecret: z25.string().optional().describe("The secret of the app in Dev Center")
+});
+var CreateProjectInputSchema = ProjectSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateProjectInputSchema = CreateProjectInputSchema.partial();
+
+// src/template/template.ts
+import { z as z26 } from "zod";
+var TemplateSchema = TenantEntitySchema.extend({
+  /** URL to download the template from */
+  downloadUrl: z26.url()
+});
+var CreateTemplateInputSchema = TemplateSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
+export {
+  AVAILABLE_MODELS,
+  AVAILABLE_MODELS_MAP,
+  AgentSchema,
+  AllowedCommands,
+  ApiCallSchema,
+  AssertionResultSchema,
+  AssertionResultStatus,
+  BaseEntitySchema,
+  BaseTestSchema,
+  BuildCheckTestSchema,
+  CommandExecutionSchema,
+  CommandExecutionTestSchema,
+  CreateAgentInputSchema,
+  CreateEvalRunInputSchema,
+  CreateProjectInputSchema,
+  CreateSkillInputSchema,
+  CreateSkillsGroupInputSchema,
+  CreateTemplateInputSchema,
+  CreateTestScenarioInputSchema,
+  CreateTestSuiteInputSchema,
+  DiffContentSchema,
+  DiffLineSchema,
+  DiffLineTypeSchema,
+  EnvironmentSchema,
+  EvalMetricsSchema,
+  EvalRunResultSchema,
+  EvalRunSchema,
+  EvalStatus,
+  EvalStatusSchema,
+  EvaluationLogSchema,
+  EvaluationProgressSchema,
+  EvaluationResultSchema,
+  ExecutionTraceSchema,
+  ExpectedFileSchema,
+  FailureAnalysisSchema,
+  FailureCategory,
+  FailureSeverity,
+  FileContentCheckSchema,
+  FileContentTestSchema,
+  FileModificationSchema,
+  FilePresenceTestSchema,
+  LLMBreakdownStatsSchema,
+  LLMStepType,
+  LLMTestSchema,
+  LLMTraceSchema,
+  LLMTraceStepSchema,
+  LLMTraceSummarySchema,
+  LLM_TIMEOUT,
+  LeanEvaluationResultSchema,
+  LiveTraceEventSchema,
+  LiveTraceEventType,
+  LocalProjectConfigSchema,
+  MCPServerConfigSchema,
+  MetaSiteConfigSchema,
+  ModelConfigSchema,
+  ModelIds,
+  ModelIdsSchema,
+  ModelPricingSchema,
+  ModelSchema,
+  PlaywrightNLTestSchema,
+  ProjectSchema,
+  PromptResultSchema,
+  SiteConfigTestSchema,
+  SkillMetadataSchema,
+  SkillSchema,
+  SkillVersionSchema,
+  SkillsGroupSchema,
+  TRACE_EVENT_PREFIX,
+  TargetSchema,
+  TemplateSchema,
+  TenantEntitySchema,
+  TestImportance,
+  TestImportanceSchema,
+  TestScenarioSchema,
+  TestSchema,
+  TestSuiteSchema,
+  TestType,
+  TestTypeSchema,
+  TokenUsageSchema,
+  ToolTestSchema,
+  TriggerMetadataSchema,
+  TriggerSchema,
+  TriggerType,
+  UpdateAgentInputSchema,
+  UpdateProjectInputSchema,
+  UpdateSkillInputSchema,
+  UpdateSkillsGroupInputSchema,
+  UpdateTemplateInputSchema,
+  UpdateTestScenarioInputSchema,
+  UpdateTestSuiteInputSchema,
+  VitestTestSchema,
+  formatTraceEventLine,
+  parseTraceEventLine
+};
+//# sourceMappingURL=index.mjs.map