@wix/evalforge-types 0.3.0

package/build/index.js

@@ -0,0 +1,1047 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  AVAILABLE_MODELS: () => AVAILABLE_MODELS,
+  AVAILABLE_MODELS_MAP: () => AVAILABLE_MODELS_MAP,
+  AgentSchema: () => AgentSchema,
+  AllowedCommands: () => AllowedCommands,
+  ApiCallSchema: () => ApiCallSchema,
+  AssertionResultSchema: () => AssertionResultSchema,
+  AssertionResultStatus: () => AssertionResultStatus,
+  BaseEntitySchema: () => BaseEntitySchema,
+  BaseTestSchema: () => BaseTestSchema,
+  BuildCheckTestSchema: () => BuildCheckTestSchema,
+  CommandExecutionSchema: () => CommandExecutionSchema,
+  CommandExecutionTestSchema: () => CommandExecutionTestSchema,
+  CreateAgentInputSchema: () => CreateAgentInputSchema,
+  CreateEvalRunInputSchema: () => CreateEvalRunInputSchema,
+  CreateProjectInputSchema: () => CreateProjectInputSchema,
+  CreateSkillInputSchema: () => CreateSkillInputSchema,
+  CreateSkillsGroupInputSchema: () => CreateSkillsGroupInputSchema,
+  CreateTemplateInputSchema: () => CreateTemplateInputSchema,
+  CreateTestScenarioInputSchema: () => CreateTestScenarioInputSchema,
+  CreateTestSuiteInputSchema: () => CreateTestSuiteInputSchema,
+  DiffContentSchema: () => DiffContentSchema,
+  DiffLineSchema: () => DiffLineSchema,
+  DiffLineTypeSchema: () => DiffLineTypeSchema,
+  EnvironmentSchema: () => EnvironmentSchema,
+  EvalMetricsSchema: () => EvalMetricsSchema,
+  EvalRunResultSchema: () => EvalRunResultSchema,
+  EvalRunSchema: () => EvalRunSchema,
+  EvalStatus: () => EvalStatus,
+  EvalStatusSchema: () => EvalStatusSchema,
+  EvaluationLogSchema: () => EvaluationLogSchema,
+  EvaluationProgressSchema: () => EvaluationProgressSchema,
+  EvaluationResultSchema: () => EvaluationResultSchema,
+  ExecutionTraceSchema: () => ExecutionTraceSchema,
+  ExpectedFileSchema: () => ExpectedFileSchema,
+  FailureAnalysisSchema: () => FailureAnalysisSchema,
+  FailureCategory: () => FailureCategory,
+  FailureSeverity: () => FailureSeverity,
+  FileContentCheckSchema: () => FileContentCheckSchema,
+  FileContentTestSchema: () => FileContentTestSchema,
+  FileModificationSchema: () => FileModificationSchema,
+  FilePresenceTestSchema: () => FilePresenceTestSchema,
+  LLMBreakdownStatsSchema: () => LLMBreakdownStatsSchema,
+  LLMStepType: () => LLMStepType,
+  LLMTestSchema: () => LLMTestSchema,
+  LLMTraceSchema: () => LLMTraceSchema,
+  LLMTraceStepSchema: () => LLMTraceStepSchema,
+  LLMTraceSummarySchema: () => LLMTraceSummarySchema,
+  LLM_TIMEOUT: () => LLM_TIMEOUT,
+  LeanEvaluationResultSchema: () => LeanEvaluationResultSchema,
+  LiveTraceEventSchema: () => LiveTraceEventSchema,
+  LiveTraceEventType: () => LiveTraceEventType,
+  LocalProjectConfigSchema: () => LocalProjectConfigSchema,
+  MCPServerConfigSchema: () => MCPServerConfigSchema,
+  MetaSiteConfigSchema: () => MetaSiteConfigSchema,
+  ModelConfigSchema: () => ModelConfigSchema,
+  ModelIds: () => ModelIds,
+  ModelIdsSchema: () => ModelIdsSchema,
+  ModelPricingSchema: () => ModelPricingSchema,
+  ModelSchema: () => ModelSchema,
+  PlaywrightNLTestSchema: () => PlaywrightNLTestSchema,
+  ProjectSchema: () => ProjectSchema,
+  PromptResultSchema: () => PromptResultSchema,
+  SiteConfigTestSchema: () => SiteConfigTestSchema,
+  SkillMetadataSchema: () => SkillMetadataSchema,
+  SkillSchema: () => SkillSchema,
+  SkillVersionSchema: () => SkillVersionSchema,
+  SkillsGroupSchema: () => SkillsGroupSchema,
+  TRACE_EVENT_PREFIX: () => TRACE_EVENT_PREFIX,
+  TargetSchema: () => TargetSchema,
+  TemplateSchema: () => TemplateSchema,
+  TenantEntitySchema: () => TenantEntitySchema,
+  TestImportance: () => TestImportance,
+  TestImportanceSchema: () => TestImportanceSchema,
+  TestScenarioSchema: () => TestScenarioSchema,
+  TestSchema: () => TestSchema,
+  TestSuiteSchema: () => TestSuiteSchema,
+  TestType: () => TestType,
+  TestTypeSchema: () => TestTypeSchema,
+  TokenUsageSchema: () => TokenUsageSchema,
+  ToolTestSchema: () => ToolTestSchema,
+  TriggerMetadataSchema: () => TriggerMetadataSchema,
+  TriggerSchema: () => TriggerSchema,
+  TriggerType: () => TriggerType,
+  UpdateAgentInputSchema: () => UpdateAgentInputSchema,
+  UpdateProjectInputSchema: () => UpdateProjectInputSchema,
+  UpdateSkillInputSchema: () => UpdateSkillInputSchema,
+  UpdateSkillsGroupInputSchema: () => UpdateSkillsGroupInputSchema,
+  UpdateTemplateInputSchema: () => UpdateTemplateInputSchema,
+  UpdateTestScenarioInputSchema: () => UpdateTestScenarioInputSchema,
+  UpdateTestSuiteInputSchema: () => UpdateTestSuiteInputSchema,
+  VitestTestSchema: () => VitestTestSchema,
+  formatTraceEventLine: () => formatTraceEventLine,
+  parseTraceEventLine: () => parseTraceEventLine
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/common/base-entity.ts
+var import_zod = require("zod");
+var BaseEntitySchema = import_zod.z.object({
+  id: import_zod.z.string(),
+  name: import_zod.z.string().min(1),
+  description: import_zod.z.string(),
+  createdAt: import_zod.z.string(),
+  updatedAt: import_zod.z.string(),
+  deleted: import_zod.z.boolean().optional()
+});
+var TenantEntitySchema = BaseEntitySchema.extend({
+  projectId: import_zod.z.string()
+});
+
+// src/common/mcp.ts
+var import_zod2 = require("zod");
+var MCPServerConfigSchema = import_zod2.z.object({
+  /** Unique name for this MCP server */
+  name: import_zod2.z.string(),
+  /** Command to start the MCP server */
+  command: import_zod2.z.string(),
+  /** Command line arguments */
+  args: import_zod2.z.array(import_zod2.z.string()).optional(),
+  /** Environment variables for the server process */
+  envVars: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.string()).optional(),
+  /** Tools to disable for this MCP server */
+  disabledTools: import_zod2.z.array(import_zod2.z.string()).optional()
+});
+
+// src/common/models.ts
+var import_zod3 = require("zod");
+var ModelIds = /* @__PURE__ */ ((ModelIds2) => {
+  ModelIds2["CLAUDE_3_HAIKU_1_0"] = "CLAUDE_3_HAIKU_1_0";
+  ModelIds2["CLAUDE_3_OPUS_1_0"] = "CLAUDE_3_OPUS_1_0";
+  ModelIds2["CLAUDE_3_SONNET_1_0"] = "CLAUDE_3_SONNET_1_0";
+  ModelIds2["CLAUDE_3_5_SONNET_1_0"] = "CLAUDE_3_5_SONNET_1_0";
+  ModelIds2["CLAUDE_3_5_SONNET_2_0"] = "CLAUDE_3_5_SONNET_2_0";
+  ModelIds2["CLAUDE_3_7_SONNET_1_0"] = "CLAUDE_3_7_SONNET_1_0";
+  ModelIds2["CLAUDE_4_OPUS_1_0"] = "CLAUDE_4_OPUS_1_0";
+  ModelIds2["CLAUDE_4_SONNET_1_0"] = "CLAUDE_4_SONNET_1_0";
+  return ModelIds2;
+})(ModelIds || {});
+var ModelIdsSchema = import_zod3.z.enum(ModelIds);
+var ModelConfigSchema = import_zod3.z.object({
+  model: ModelIdsSchema,
+  temperature: import_zod3.z.number().min(0).max(1).optional(),
+  maxTokens: import_zod3.z.number().min(1).optional()
+});
+var ModelPricingSchema = import_zod3.z.object({
+  inputPer1M: import_zod3.z.number(),
+  outputPer1M: import_zod3.z.number()
+});
+var ModelSchema = import_zod3.z.object({
+  /** AI Gateway model ID */
+  id: ModelIdsSchema,
+  /** Display name */
+  name: import_zod3.z.string(),
+  /** Provider (always 'anthropic') */
+  provider: import_zod3.z.literal("anthropic"),
+  /** Provider's model identifier (e.g., "claude-3-5-sonnet-20241022") */
+  providerModelId: import_zod3.z.string(),
+  /** Pricing per 1M tokens */
+  pricing: ModelPricingSchema
+});
+var AVAILABLE_MODELS = [
+  {
+    id: "CLAUDE_4_SONNET_1_0" /* CLAUDE_4_SONNET_1_0 */,
+    name: "Claude 4 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-4-sonnet",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_4_OPUS_1_0" /* CLAUDE_4_OPUS_1_0 */,
+    name: "Claude 4 Opus",
+    provider: "anthropic",
+    providerModelId: "claude-4-opus",
+    pricing: { inputPer1M: 15, outputPer1M: 75 }
+  },
+  {
+    id: "CLAUDE_3_7_SONNET_1_0" /* CLAUDE_3_7_SONNET_1_0 */,
+    name: "Claude 3.7 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-3-7-sonnet",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_5_SONNET_2_0" /* CLAUDE_3_5_SONNET_2_0 */,
+    name: "Claude 3.5 Sonnet v2",
+    provider: "anthropic",
+    providerModelId: "claude-3-5-sonnet-20241022",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_5_SONNET_1_0" /* CLAUDE_3_5_SONNET_1_0 */,
+    name: "Claude 3.5 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-3-5-sonnet-20240620",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_OPUS_1_0" /* CLAUDE_3_OPUS_1_0 */,
+    name: "Claude 3 Opus",
+    provider: "anthropic",
+    providerModelId: "claude-3-opus-20240229",
+    pricing: { inputPer1M: 15, outputPer1M: 75 }
+  },
+  {
+    id: "CLAUDE_3_SONNET_1_0" /* CLAUDE_3_SONNET_1_0 */,
+    name: "Claude 3 Sonnet",
+    provider: "anthropic",
+    providerModelId: "claude-3-sonnet-20240229",
+    pricing: { inputPer1M: 3, outputPer1M: 15 }
+  },
+  {
+    id: "CLAUDE_3_HAIKU_1_0" /* CLAUDE_3_HAIKU_1_0 */,
+    name: "Claude 3 Haiku",
+    provider: "anthropic",
+    providerModelId: "claude-3-haiku-20240307",
+    pricing: { inputPer1M: 0.25, outputPer1M: 1.25 }
+  }
+];
+var AVAILABLE_MODELS_MAP = Object.fromEntries(
+  AVAILABLE_MODELS.map((model) => [model.id, model])
+);
+
+// src/target/target.ts
+var TargetSchema = TenantEntitySchema.extend({
+  // Base for all testable entities
+  // Specific targets add their own fields
+});
+
+// src/target/agent.ts
+var import_zod4 = require("zod");
+var AgentSchema = TargetSchema.extend({
+  /** Command to run the agent */
+  runCommand: import_zod4.z.string(),
+  /** Optional model configuration override */
+  modelConfig: ModelConfigSchema.optional()
+});
+var CreateAgentInputSchema = AgentSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateAgentInputSchema = CreateAgentInputSchema.partial();
+
+// src/target/skill.ts
+var import_zod5 = require("zod");
+var SkillMetadataSchema = import_zod5.z.object({
+  name: import_zod5.z.string(),
+  description: import_zod5.z.string(),
+  allowedTools: import_zod5.z.array(import_zod5.z.string()).optional(),
+  skills: import_zod5.z.array(import_zod5.z.string()).optional()
+});
+var SkillVersionSchema = import_zod5.z.object({
+  id: import_zod5.z.string(),
+  skillId: import_zod5.z.string(),
+  skillMd: import_zod5.z.string(),
+  metadata: SkillMetadataSchema,
+  model: ModelConfigSchema.optional(),
+  systemPrompt: import_zod5.z.string().optional(),
+  version: import_zod5.z.number(),
+  createdAt: import_zod5.z.string(),
+  notes: import_zod5.z.string().optional()
+});
+var SkillSchema = TargetSchema.extend({
+  /** The current SKILL.md content */
+  skillMd: import_zod5.z.string()
+});
+var CreateSkillInputSchema = SkillSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateSkillInputSchema = CreateSkillInputSchema.partial();
+
+// src/target/skills-group.ts
+var import_zod6 = require("zod");
+var SkillsGroupSchema = TenantEntitySchema.extend({
+  /** IDs of skills in this group */
+  skillIds: import_zod6.z.array(import_zod6.z.string())
+});
+var CreateSkillsGroupInputSchema = SkillsGroupSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateSkillsGroupInputSchema = CreateSkillsGroupInputSchema.partial();
+
+// src/test/index.ts
+var import_zod17 = require("zod");
+
+// src/test/base.ts
+var import_zod7 = require("zod");
+var TestType = /* @__PURE__ */ ((TestType2) => {
+  TestType2["LLM"] = "LLM";
+  TestType2["TOOL"] = "TOOL";
+  TestType2["SITE_CONFIG"] = "SITE_CONFIG";
+  TestType2["COMMAND_EXECUTION"] = "COMMAND_EXECUTION";
+  TestType2["FILE_PRESENCE"] = "FILE_PRESENCE";
+  TestType2["FILE_CONTENT"] = "FILE_CONTENT";
+  TestType2["BUILD_CHECK"] = "BUILD_CHECK";
+  TestType2["VITEST"] = "VITEST";
+  TestType2["PLAYWRIGHT_NL"] = "PLAYWRIGHT_NL";
+  return TestType2;
+})(TestType || {});
+var TestTypeSchema = import_zod7.z.enum(TestType);
+var TestImportance = /* @__PURE__ */ ((TestImportance2) => {
+  TestImportance2["LOW"] = "low";
+  TestImportance2["MEDIUM"] = "medium";
+  TestImportance2["HIGH"] = "high";
+  TestImportance2["CRITICAL"] = "critical";
+  return TestImportance2;
+})(TestImportance || {});
+var TestImportanceSchema = import_zod7.z.enum(TestImportance);
+var BaseTestSchema = import_zod7.z.object({
+  id: import_zod7.z.string(),
+  type: TestTypeSchema,
+  name: import_zod7.z.string().min(3),
+  description: import_zod7.z.string().optional(),
+  importance: TestImportanceSchema.optional()
+});
+
+// src/test/llm.ts
+var import_zod8 = require("zod");
+var LLMTestSchema = BaseTestSchema.extend({
+  type: import_zod8.z.literal("LLM" /* LLM */),
+  /** Maximum steps for the LLM to take */
+  maxSteps: import_zod8.z.number().min(1).max(100),
+  /** Prompt to send to the evaluator */
+  prompt: import_zod8.z.string().min(1),
+  /** ID of the evaluator agent to use */
+  evaluatorId: import_zod8.z.string()
+});
+
+// src/test/tool.ts
+var import_zod9 = require("zod");
+var ToolTestSchema = BaseTestSchema.extend({
+  type: import_zod9.z.literal("TOOL" /* TOOL */),
+  /** Name of the tool that should be called */
+  toolName: import_zod9.z.string().min(3),
+  /** Expected arguments for the tool call */
+  args: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.any()),
+  /** Expected content in the tool results */
+  resultsContent: import_zod9.z.string()
+});
+
+// src/test/site-config.ts
+var import_zod10 = require("zod");
+var SiteConfigTestSchema = BaseTestSchema.extend({
+  type: import_zod10.z.literal("SITE_CONFIG" /* SITE_CONFIG */),
+  /** URL to call */
+  url: import_zod10.z.string().url(),
+  /** HTTP method */
+  method: import_zod10.z.enum(["GET", "POST"]),
+  /** Request body (for POST) */
+  body: import_zod10.z.string().optional(),
+  /** Expected HTTP status code */
+  expectedStatusCode: import_zod10.z.number().int().min(100).max(599),
+  /** Expected response content */
+  expectedResponse: import_zod10.z.string().optional(),
+  /** JMESPath expression to extract from response */
+  expectedResponseJMESPath: import_zod10.z.string().optional()
+});
+
+// src/test/command-execution.ts
+var import_zod11 = require("zod");
+var AllowedCommands = [
+  "yarn install --no-immutable && yarn build",
+  "npm run build",
+  "yarn typecheck"
+];
+var CommandExecutionTestSchema = BaseTestSchema.extend({
+  type: import_zod11.z.literal("COMMAND_EXECUTION" /* COMMAND_EXECUTION */),
+  /** Command to execute (must be in AllowedCommands) */
+  command: import_zod11.z.string().refine((value) => AllowedCommands.includes(value), {
+    message: `Command must be one of: ${AllowedCommands.join(", ")}`
+  }),
+  /** Expected exit code (default: 0) */
+  expectedExitCode: import_zod11.z.number().default(0).optional()
+});
+
+// src/test/file-presence.ts
+var import_zod12 = require("zod");
+var FilePresenceTestSchema = BaseTestSchema.extend({
+  type: import_zod12.z.literal("FILE_PRESENCE" /* FILE_PRESENCE */),
+  /** Paths to check */
+  paths: import_zod12.z.array(import_zod12.z.string()),
+  /** Whether files should exist (true) or not exist (false) */
+  shouldExist: import_zod12.z.boolean()
+});
+
+// src/test/file-content.ts
+var import_zod13 = require("zod");
+var FileContentCheckSchema = import_zod13.z.object({
+  /** Strings that must be present in the file */
+  contains: import_zod13.z.array(import_zod13.z.string()).optional(),
+  /** Strings that must NOT be present in the file */
+  notContains: import_zod13.z.array(import_zod13.z.string()).optional(),
+  /** Regex pattern the content must match */
+  matches: import_zod13.z.string().optional(),
+  /** JSON path checks for structured content */
+  jsonPath: import_zod13.z.array(
+    import_zod13.z.object({
+      path: import_zod13.z.string(),
+      value: import_zod13.z.unknown()
+    })
+  ).optional(),
+  /** Lines that should be added (for diff checking) */
+  added: import_zod13.z.array(import_zod13.z.string()).optional(),
+  /** Lines that should be removed (for diff checking) */
+  removed: import_zod13.z.array(import_zod13.z.string()).optional()
+});
+var FileContentTestSchema = BaseTestSchema.extend({
+  type: import_zod13.z.literal("FILE_CONTENT" /* FILE_CONTENT */),
+  /** Path to the file to check */
+  path: import_zod13.z.string(),
+  /** Content checks to perform */
+  checks: FileContentCheckSchema
+});
+
+// src/test/build-check.ts
+var import_zod14 = require("zod");
+var BuildCheckTestSchema = BaseTestSchema.extend({
+  type: import_zod14.z.literal("BUILD_CHECK" /* BUILD_CHECK */),
+  /** Build command to execute */
+  command: import_zod14.z.string(),
+  /** Whether the build should succeed */
+  expectSuccess: import_zod14.z.boolean(),
+  /** Maximum allowed warnings (optional) */
+  allowedWarnings: import_zod14.z.number().optional(),
+  /** Timeout in milliseconds */
+  timeout: import_zod14.z.number().optional()
+});
+
+// src/test/vitest.ts
+var import_zod15 = require("zod");
+var VitestTestSchema = BaseTestSchema.extend({
+  type: import_zod15.z.literal("VITEST" /* VITEST */),
+  /** Test file content */
+  testFile: import_zod15.z.string(),
+  /** Name of the test file */
+  testFileName: import_zod15.z.string(),
+  /** Minimum pass rate required (0-100) */
+  minPassRate: import_zod15.z.number().min(0).max(100)
+});
+
+// src/test/playwright-nl.ts
+var import_zod16 = require("zod");
+var PlaywrightNLTestSchema = BaseTestSchema.extend({
+  type: import_zod16.z.literal("PLAYWRIGHT_NL" /* PLAYWRIGHT_NL */),
+  /** Natural language steps to execute */
+  steps: import_zod16.z.array(import_zod16.z.string()),
+  /** Expected outcome description */
+  expectedOutcome: import_zod16.z.string(),
+  /** Timeout in milliseconds */
+  timeout: import_zod16.z.number().optional()
+});
+
+// src/test/index.ts
+var TestSchema = import_zod17.z.discriminatedUnion("type", [
+  LLMTestSchema,
+  ToolTestSchema,
+  SiteConfigTestSchema,
+  CommandExecutionTestSchema,
+  FilePresenceTestSchema,
+  FileContentTestSchema,
+  BuildCheckTestSchema,
+  VitestTestSchema,
+  PlaywrightNLTestSchema
+]);
+
+// src/scenario/environment.ts
+var import_zod18 = require("zod");
+var LocalProjectConfigSchema = import_zod18.z.object({
+  /** Template ID to use for the local project */
+  templateId: import_zod18.z.string().optional(),
+  /** Files to create in the project */
+  files: import_zod18.z.array(
+    import_zod18.z.object({
+      path: import_zod18.z.string().min(1),
+      content: import_zod18.z.string().min(1)
+    })
+  ).optional()
+});
+var MetaSiteConfigSchema = import_zod18.z.object({
+  configurations: import_zod18.z.array(
+    import_zod18.z.object({
+      name: import_zod18.z.string().min(1),
+      apiCalls: import_zod18.z.array(
+        import_zod18.z.object({
+          url: import_zod18.z.string().url(),
+          method: import_zod18.z.enum(["POST", "PUT"]),
+          body: import_zod18.z.string()
+        })
+      )
+    })
+  ).optional()
+});
+var EnvironmentSchema = import_zod18.z.object({
+  /** Local project configuration */
+  localProject: LocalProjectConfigSchema.optional(),
+  /** Meta site configuration */
+  metaSite: MetaSiteConfigSchema.optional()
+});
+
+// src/scenario/test-scenario.ts
+var import_zod19 = require("zod");
+var ExpectedFileSchema = import_zod19.z.object({
+  /** Relative path where the file should be created */
+  path: import_zod19.z.string(),
+  /** Optional expected content */
+  content: import_zod19.z.string().optional()
+});
+var TestScenarioSchema = TenantEntitySchema.extend({
+  /** The prompt sent to the agent to trigger the task */
+  triggerPrompt: import_zod19.z.string().min(10),
+  /** ID of the template to use for this scenario */
+  templateId: import_zod19.z.string().optional()
+});
+var CreateTestScenarioInputSchema = TestScenarioSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateTestScenarioInputSchema = CreateTestScenarioInputSchema.partial();
+
+// src/suite/test-suite.ts
+var import_zod20 = require("zod");
+var TestSuiteSchema = TenantEntitySchema.extend({
+  /** IDs of test scenarios in this suite */
+  scenarioIds: import_zod20.z.array(import_zod20.z.string())
+});
+var CreateTestSuiteInputSchema = TestSuiteSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateTestSuiteInputSchema = CreateTestSuiteInputSchema.partial();
+
+// src/evaluation/metrics.ts
+var import_zod21 = require("zod");
+var TokenUsageSchema = import_zod21.z.object({
+  prompt: import_zod21.z.number(),
+  completion: import_zod21.z.number(),
+  total: import_zod21.z.number()
+});
+var EvalMetricsSchema = import_zod21.z.object({
+  totalAssertions: import_zod21.z.number(),
+  passed: import_zod21.z.number(),
+  failed: import_zod21.z.number(),
+  skipped: import_zod21.z.number(),
+  errors: import_zod21.z.number(),
+  passRate: import_zod21.z.number(),
+  avgDuration: import_zod21.z.number(),
+  totalDuration: import_zod21.z.number()
+});
+var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
+  EvalStatus2["PENDING"] = "pending";
+  EvalStatus2["RUNNING"] = "running";
+  EvalStatus2["COMPLETED"] = "completed";
+  EvalStatus2["FAILED"] = "failed";
+  EvalStatus2["CANCELLED"] = "cancelled";
+  return EvalStatus2;
+})(EvalStatus || {});
+var EvalStatusSchema = import_zod21.z.enum(EvalStatus);
+var LLMStepType = /* @__PURE__ */ ((LLMStepType2) => {
+  LLMStepType2["COMPLETION"] = "completion";
+  LLMStepType2["TOOL_USE"] = "tool_use";
+  LLMStepType2["TOOL_RESULT"] = "tool_result";
+  LLMStepType2["THINKING"] = "thinking";
+  return LLMStepType2;
+})(LLMStepType || {});
+var LLMTraceStepSchema = import_zod21.z.object({
+  id: import_zod21.z.string(),
+  stepNumber: import_zod21.z.number(),
+  type: import_zod21.z.enum(LLMStepType),
+  model: import_zod21.z.string(),
+  provider: import_zod21.z.string(),
+  startedAt: import_zod21.z.string(),
+  durationMs: import_zod21.z.number(),
+  tokenUsage: TokenUsageSchema,
+  costUsd: import_zod21.z.number(),
+  toolName: import_zod21.z.string().optional(),
+  toolArguments: import_zod21.z.string().optional(),
+  inputPreview: import_zod21.z.string().optional(),
+  outputPreview: import_zod21.z.string().optional(),
+  success: import_zod21.z.boolean(),
+  error: import_zod21.z.string().optional()
+});
+var LLMBreakdownStatsSchema = import_zod21.z.object({
+  count: import_zod21.z.number(),
+  durationMs: import_zod21.z.number(),
+  tokens: import_zod21.z.number(),
+  costUsd: import_zod21.z.number()
+});
+var LLMTraceSummarySchema = import_zod21.z.object({
+  totalSteps: import_zod21.z.number(),
+  totalDurationMs: import_zod21.z.number(),
+  totalTokens: TokenUsageSchema,
+  totalCostUsd: import_zod21.z.number(),
+  stepTypeBreakdown: import_zod21.z.record(import_zod21.z.string(), LLMBreakdownStatsSchema).optional(),
+  modelBreakdown: import_zod21.z.record(import_zod21.z.string(), LLMBreakdownStatsSchema),
+  modelsUsed: import_zod21.z.array(import_zod21.z.string())
+});
+var LLMTraceSchema = import_zod21.z.object({
+  id: import_zod21.z.string(),
+  steps: import_zod21.z.array(LLMTraceStepSchema),
+  summary: LLMTraceSummarySchema
+});
+
+// src/evaluation/eval-result.ts
+var import_zod22 = require("zod");
+var AssertionResultStatus = /* @__PURE__ */ ((AssertionResultStatus2) => {
+  AssertionResultStatus2["PASSED"] = "passed";
+  AssertionResultStatus2["FAILED"] = "failed";
+  AssertionResultStatus2["SKIPPED"] = "skipped";
+  AssertionResultStatus2["ERROR"] = "error";
+  return AssertionResultStatus2;
+})(AssertionResultStatus || {});
+var AssertionResultSchema = import_zod22.z.object({
+  id: import_zod22.z.string(),
+  assertionId: import_zod22.z.string(),
+  assertionType: import_zod22.z.string(),
+  assertionName: import_zod22.z.string(),
+  status: import_zod22.z.enum(AssertionResultStatus),
+  message: import_zod22.z.string().optional(),
+  expected: import_zod22.z.string().optional(),
+  actual: import_zod22.z.string().optional(),
+  duration: import_zod22.z.number().optional(),
+  details: import_zod22.z.record(import_zod22.z.string(), import_zod22.z.unknown()).optional(),
+  llmTraceSteps: import_zod22.z.array(LLMTraceStepSchema).optional()
+});
+var EvalRunResultSchema = import_zod22.z.object({
+  id: import_zod22.z.string(),
+  targetId: import_zod22.z.string(),
+  targetName: import_zod22.z.string().optional(),
+  scenarioId: import_zod22.z.string(),
+  scenarioName: import_zod22.z.string(),
+  modelConfig: ModelConfigSchema.optional(),
+  assertionResults: import_zod22.z.array(AssertionResultSchema),
+  metrics: EvalMetricsSchema.optional(),
+  passed: import_zod22.z.number(),
+  failed: import_zod22.z.number(),
+  passRate: import_zod22.z.number(),
+  duration: import_zod22.z.number(),
+  outputText: import_zod22.z.string().optional(),
+  files: import_zod22.z.array(ExpectedFileSchema).optional(),
+  startedAt: import_zod22.z.string().optional(),
+  completedAt: import_zod22.z.string().optional(),
+  llmTrace: LLMTraceSchema.optional()
+});
+var PromptResultSchema = import_zod22.z.object({
+  text: import_zod22.z.string(),
+  files: import_zod22.z.array(import_zod22.z.unknown()).optional(),
+  finishReason: import_zod22.z.string().optional(),
+  reasoning: import_zod22.z.string().optional(),
+  reasoningDetails: import_zod22.z.unknown().optional(),
+  toolCalls: import_zod22.z.array(import_zod22.z.unknown()).optional(),
+  toolResults: import_zod22.z.array(import_zod22.z.unknown()).optional(),
+  warnings: import_zod22.z.array(import_zod22.z.unknown()).optional(),
+  sources: import_zod22.z.array(import_zod22.z.unknown()).optional(),
+  steps: import_zod22.z.array(import_zod22.z.unknown()),
+  generationTimeMs: import_zod22.z.number(),
+  prompt: import_zod22.z.string(),
+  systemPrompt: import_zod22.z.string(),
+  usage: import_zod22.z.object({
+    totalTokens: import_zod22.z.number().optional(),
+    totalMicrocentsSpent: import_zod22.z.number().optional()
+  })
+});
+var EvaluationResultSchema = import_zod22.z.object({
+  id: import_zod22.z.string(),
+  runId: import_zod22.z.string(),
+  timestamp: import_zod22.z.number(),
+  promptResult: PromptResultSchema,
+  testResults: import_zod22.z.array(import_zod22.z.unknown()),
+  tags: import_zod22.z.array(import_zod22.z.string()).optional(),
+  feedback: import_zod22.z.string().optional(),
+  score: import_zod22.z.number(),
+  suiteId: import_zod22.z.string().optional()
+});
+var LeanEvaluationResultSchema = import_zod22.z.object({
+  id: import_zod22.z.string(),
+  runId: import_zod22.z.string(),
+  timestamp: import_zod22.z.number(),
+  tags: import_zod22.z.array(import_zod22.z.string()).optional(),
+  scenarioId: import_zod22.z.string(),
+  scenarioVersion: import_zod22.z.number().optional(),
+  targetId: import_zod22.z.string(),
+  targetVersion: import_zod22.z.number().optional(),
+  suiteId: import_zod22.z.string().optional(),
+  score: import_zod22.z.number(),
+  time: import_zod22.z.number().optional(),
+  microcentsSpent: import_zod22.z.number().optional()
+});
+
+// src/evaluation/eval-run.ts
+var import_zod24 = require("zod");
+
+// src/evaluation/live-trace.ts
+var import_zod23 = require("zod");
+var LiveTraceEventType = /* @__PURE__ */ ((LiveTraceEventType2) => {
+  LiveTraceEventType2["THINKING"] = "thinking";
+  LiveTraceEventType2["TOOL_USE"] = "tool_use";
+  LiveTraceEventType2["COMPLETION"] = "completion";
+  LiveTraceEventType2["TOOL_RESULT"] = "tool_result";
+  return LiveTraceEventType2;
+})(LiveTraceEventType || {});
+var LiveTraceEventSchema = import_zod23.z.object({
+  /** The evaluation run ID */
+  evalRunId: import_zod23.z.string(),
+  /** The scenario ID being executed */
+  scenarioId: import_zod23.z.string(),
+  /** The scenario name for display */
+  scenarioName: import_zod23.z.string(),
+  /** The target ID (skill, agent, etc.) */
+  targetId: import_zod23.z.string(),
+  /** The target name for display */
+  targetName: import_zod23.z.string(),
+  /** Step number in the current scenario execution */
+  stepNumber: import_zod23.z.number(),
+  /** Type of trace event */
+  type: import_zod23.z.enum(LiveTraceEventType),
+  /** Tool name if this is a tool_use event */
+  toolName: import_zod23.z.string().optional(),
+  /** Tool arguments preview (truncated JSON) */
+  toolArgs: import_zod23.z.string().optional(),
+  /** Output preview (truncated text) */
+  outputPreview: import_zod23.z.string().optional(),
+  /** Timestamp when this event occurred */
+  timestamp: import_zod23.z.string(),
+  /** Whether this is the final event for this scenario */
+  isComplete: import_zod23.z.boolean()
+});
+var TRACE_EVENT_PREFIX = "TRACE_EVENT:";
+function parseTraceEventLine(line) {
+  if (!line.startsWith(TRACE_EVENT_PREFIX)) {
+    return null;
+  }
+  try {
+    const jsonStr = line.slice(TRACE_EVENT_PREFIX.length);
+    const parsed = JSON.parse(jsonStr);
+    const result = LiveTraceEventSchema.safeParse(parsed);
+    return result.success ? result.data : null;
+  } catch {
+    return null;
+  }
+}
+function formatTraceEventLine(event) {
+  return `${TRACE_EVENT_PREFIX}${JSON.stringify(event)}`;
+}
+
+// src/evaluation/eval-run.ts
+var TriggerType = /* @__PURE__ */ ((TriggerType2) => {
+  TriggerType2["RESOURCES_UPDATED"] = "RESOURCES_UPDATED";
+  TriggerType2["MCP_VERSION_RELEASE"] = "MCP_VERSION_RELEASE";
+  TriggerType2["MCP_PREVIEW_CREATED"] = "MCP_PREVIEW_CREATED";
+  TriggerType2["MANUAL"] = "MANUAL";
+  return TriggerType2;
+})(TriggerType || {});
+var TriggerMetadataSchema = import_zod24.z.object({
+  version: import_zod24.z.string().optional(),
+  resourceUpdated: import_zod24.z.array(import_zod24.z.string()).optional()
+});
+var TriggerSchema = import_zod24.z.object({
+  id: import_zod24.z.string(),
+  metadata: TriggerMetadataSchema.optional(),
+  type: import_zod24.z.enum(TriggerType)
+});
+var FailureCategory = /* @__PURE__ */ ((FailureCategory2) => {
+  FailureCategory2["MISSING_FILE"] = "missing_file";
+  FailureCategory2["WRONG_CONTENT"] = "wrong_content";
+  FailureCategory2["BUILD_ERROR"] = "build_error";
+  FailureCategory2["TEST_FAILURE"] = "test_failure";
+  FailureCategory2["RUNTIME_ERROR"] = "runtime_error";
+  FailureCategory2["PERFORMANCE"] = "performance";
+  return FailureCategory2;
+})(FailureCategory || {});
+var FailureSeverity = /* @__PURE__ */ ((FailureSeverity2) => {
+  FailureSeverity2["CRITICAL"] = "critical";
+  FailureSeverity2["HIGH"] = "high";
+  FailureSeverity2["MEDIUM"] = "medium";
+  FailureSeverity2["LOW"] = "low";
+  return FailureSeverity2;
+})(FailureSeverity || {});
+var DiffLineTypeSchema = import_zod24.z.enum(["added", "removed", "unchanged"]);
+var DiffLineSchema = import_zod24.z.object({
+  type: DiffLineTypeSchema,
+  content: import_zod24.z.string(),
+  lineNumber: import_zod24.z.number()
+});
+var DiffContentSchema = import_zod24.z.object({
+  path: import_zod24.z.string(),
+  expected: import_zod24.z.string(),
+  actual: import_zod24.z.string(),
+  diffLines: import_zod24.z.array(DiffLineSchema)
+});
+var CommandExecutionSchema = import_zod24.z.object({
+  command: import_zod24.z.string(),
+  exitCode: import_zod24.z.number(),
+  output: import_zod24.z.string().optional(),
+  duration: import_zod24.z.number()
+});
+var FileModificationSchema = import_zod24.z.object({
+  path: import_zod24.z.string(),
+  action: import_zod24.z.enum(["created", "modified", "deleted"])
+});
+var ApiCallSchema = import_zod24.z.object({
+  endpoint: import_zod24.z.string(),
+  tokensUsed: import_zod24.z.number(),
+  duration: import_zod24.z.number()
+});
+var ExecutionTraceSchema = import_zod24.z.object({
+  commands: import_zod24.z.array(CommandExecutionSchema),
+  filesModified: import_zod24.z.array(FileModificationSchema),
+  apiCalls: import_zod24.z.array(ApiCallSchema),
+  totalDuration: import_zod24.z.number()
+});
+var FailureAnalysisSchema = import_zod24.z.object({
+  category: import_zod24.z.enum(FailureCategory),
+  severity: import_zod24.z.enum(FailureSeverity),
+  summary: import_zod24.z.string(),
+  details: import_zod24.z.string(),
+  rootCause: import_zod24.z.string(),
+  suggestedFix: import_zod24.z.string(),
+  relatedAssertions: import_zod24.z.array(import_zod24.z.string()),
+  codeSnippet: import_zod24.z.string().optional(),
+  similarIssues: import_zod24.z.array(import_zod24.z.string()).optional(),
+  patternId: import_zod24.z.string().optional(),
+  // Extended fields for detailed debugging
+  diff: DiffContentSchema.optional(),
+  executionTrace: ExecutionTraceSchema.optional()
+});
+var EvalRunSchema = TenantEntitySchema.extend({
+  /** Agent ID for this run */
+  agentId: import_zod24.z.string().optional(),
+  /** Skills group ID for this run */
+  skillsGroupId: import_zod24.z.string().optional(),
+  /** Scenario IDs to run */
+  scenarioIds: import_zod24.z.array(import_zod24.z.string()),
+  /** Current status */
+  status: EvalStatusSchema,
+  /** Progress percentage (0-100) */
+  progress: import_zod24.z.number(),
+  /** Results for each scenario/target combination */
+  results: import_zod24.z.array(EvalRunResultSchema),
+  /** Aggregated metrics across all results */
+  aggregateMetrics: EvalMetricsSchema,
+  /** Failure analyses */
+  failureAnalyses: import_zod24.z.array(FailureAnalysisSchema).optional(),
+  /** Aggregated LLM trace summary */
+  llmTraceSummary: LLMTraceSummarySchema.optional(),
+  /** What triggered this run */
+  trigger: TriggerSchema.optional(),
+  /** When the run started (set when evaluation is triggered) */
+  startedAt: import_zod24.z.string().optional(),
+  /** When the run completed */
+  completedAt: import_zod24.z.string().optional(),
+  /** Live trace events captured during execution (for playback on results page) */
+  liveTraceEvents: import_zod24.z.array(LiveTraceEventSchema).optional()
+});
+var CreateEvalRunInputSchema = EvalRunSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  status: true,
+  progress: true,
+  results: true,
+  aggregateMetrics: true,
+  startedAt: true,
+  completedAt: true
+});
+var EvaluationProgressSchema = import_zod24.z.object({
+  runId: import_zod24.z.string(),
+  targetId: import_zod24.z.string(),
+  totalScenarios: import_zod24.z.number(),
+  completedScenarios: import_zod24.z.number(),
+  scenarioProgress: import_zod24.z.array(
+    import_zod24.z.object({
+      scenarioId: import_zod24.z.string(),
+      currentStep: import_zod24.z.string(),
+      error: import_zod24.z.string().optional()
+    })
+  ),
+  createdAt: import_zod24.z.number()
+});
+var EvaluationLogSchema = import_zod24.z.object({
+  runId: import_zod24.z.string(),
+  scenarioId: import_zod24.z.string(),
+  log: import_zod24.z.object({
+    level: import_zod24.z.enum(["info", "error", "debug"]),
+    message: import_zod24.z.string().optional(),
+    args: import_zod24.z.array(import_zod24.z.any()).optional(),
+    error: import_zod24.z.string().optional()
+  })
+});
+var LLM_TIMEOUT = 12e4;
+
+// src/project/project.ts
+var import_zod25 = require("zod");
+var ProjectSchema = BaseEntitySchema.extend({
+  appId: import_zod25.z.string().optional().describe("The ID of the app in Dev Center"),
+  appSecret: import_zod25.z.string().optional().describe("The secret of the app in Dev Center")
+});
+var CreateProjectInputSchema = ProjectSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateProjectInputSchema = CreateProjectInputSchema.partial();
+
+// src/template/template.ts
+var import_zod26 = require("zod");
+var TemplateSchema = TenantEntitySchema.extend({
+  /** URL to download the template from */
+  downloadUrl: import_zod26.z.url()
+});
+var CreateTemplateInputSchema = TemplateSchema.omit({
+  id: true,
+  createdAt: true,
+  updatedAt: true,
+  deleted: true
+});
+var UpdateTemplateInputSchema = CreateTemplateInputSchema.partial();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AVAILABLE_MODELS,
+  AVAILABLE_MODELS_MAP,
+  AgentSchema,
+  AllowedCommands,
+  ApiCallSchema,
+  AssertionResultSchema,
+  AssertionResultStatus,
+  BaseEntitySchema,
+  BaseTestSchema,
+  BuildCheckTestSchema,
+  CommandExecutionSchema,
+  CommandExecutionTestSchema,
+  CreateAgentInputSchema,
+  CreateEvalRunInputSchema,
+  CreateProjectInputSchema,
+  CreateSkillInputSchema,
+  CreateSkillsGroupInputSchema,
+  CreateTemplateInputSchema,
+  CreateTestScenarioInputSchema,
+  CreateTestSuiteInputSchema,
+  DiffContentSchema,
+  DiffLineSchema,
+  DiffLineTypeSchema,
+  EnvironmentSchema,
+  EvalMetricsSchema,
+  EvalRunResultSchema,
+  EvalRunSchema,
+  EvalStatus,
+  EvalStatusSchema,
+  EvaluationLogSchema,
+  EvaluationProgressSchema,
+  EvaluationResultSchema,
+  ExecutionTraceSchema,
+  ExpectedFileSchema,
+  FailureAnalysisSchema,
+  FailureCategory,
+  FailureSeverity,
+  FileContentCheckSchema,
+  FileContentTestSchema,
+  FileModificationSchema,
+  FilePresenceTestSchema,
+  LLMBreakdownStatsSchema,
+  LLMStepType,
+  LLMTestSchema,
+  LLMTraceSchema,
+  LLMTraceStepSchema,
+  LLMTraceSummarySchema,
+  LLM_TIMEOUT,
+  LeanEvaluationResultSchema,
+  LiveTraceEventSchema,
+  LiveTraceEventType,
+  LocalProjectConfigSchema,
+  MCPServerConfigSchema,
+  MetaSiteConfigSchema,
+  ModelConfigSchema,
+  ModelIds,
+  ModelIdsSchema,
+  ModelPricingSchema,
+  ModelSchema,
+  PlaywrightNLTestSchema,
+  ProjectSchema,
+  PromptResultSchema,
+  SiteConfigTestSchema,
+  SkillMetadataSchema,
+  SkillSchema,
+  SkillVersionSchema,
+  SkillsGroupSchema,
+  TRACE_EVENT_PREFIX,
+  TargetSchema,
+  TemplateSchema,
+  TenantEntitySchema,
+  TestImportance,
+  TestImportanceSchema,
+  TestScenarioSchema,
+  TestSchema,
+  TestSuiteSchema,
+  TestType,
+  TestTypeSchema,
+  TokenUsageSchema,
+  ToolTestSchema,
+  TriggerMetadataSchema,
+  TriggerSchema,
+  TriggerType,
+  UpdateAgentInputSchema,
+  UpdateProjectInputSchema,
+  UpdateSkillInputSchema,
+  UpdateSkillsGroupInputSchema,
+  UpdateTemplateInputSchema,
+  UpdateTestScenarioInputSchema,
+  UpdateTestSuiteInputSchema,
+  VitestTestSchema,
+  formatTraceEventLine,
+  parseTraceEventLine
+});
+//# sourceMappingURL=index.js.map