@wix/evalforge-evaluator 0.88.0 → 0.90.0

package/build/index.js

@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_evalforge_types6 = require("@wix/evalforge-types");
+var import_evalforge_types8 = require("@wix/evalforge-types");
 
 // src/config.ts
 function loadConfig() {
@@ -249,6 +249,12 @@ function applyParamsToAssertion(assertion, params) {
       }
     };
   }
+  if (assertion.type === "time_limit" && params.maxDurationMs !== void 0) {
+    return {
+      ...assertion,
+      maxDurationMs: params.maxDurationMs
+    };
+  }
   if (assertion.type === "skill_was_called" && params.skillNames !== void 0) {
     return {
       ...assertion,
@@ -274,6 +280,12 @@ function resolveSystemAssertion(assertionId, params) {
         expectedExitCode: params?.expectedExitCode ?? void 0
       };
       break;
+    case "time_limit":
+      baseAssertion = {
+        type: "time_limit",
+        maxDurationMs: params?.maxDurationMs ?? 3e5
+      };
+      break;
     case "llm_judge":
       baseAssertion = {
         type: "llm_judge",
@@ -416,7 +428,7 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
 }
 
 // src/run-scenario/index.ts
-var import_evalforge_types4 = require("@wix/evalforge-types");
+var import_evalforge_types6 = require("@wix/evalforge-types");
 var import_eval_assertions = require("@wix/eval-assertions");
 
 // src/run-scenario/environment.ts
@@ -546,7 +558,7 @@ var import_crypto2 = require("crypto");
 // src/run-scenario/agents/registry.ts
 var AgentAdapterRegistry = class {
   /**
-   * Map of command strings to their registered adapters.
+   * Map of run commands to their registered adapters.
    * Multiple commands can map to the same adapter.
    */
   adapters = /* @__PURE__ */ new Map();
@@ -575,9 +587,9 @@ var AgentAdapterRegistry = class {
     }
   }
   /**
-   * Get an adapter by command string.
+   * Get an adapter by run command.
    *
-   * @param runCommand - The command string to look up (e.g., 'claude', 'cursor')
+   * @param runCommand - The run command to look up
    * @returns The registered adapter, or undefined if not found
    */
   get(runCommand) {
@@ -586,7 +598,7 @@ var AgentAdapterRegistry = class {
   /**
    * Check if a command has a registered adapter.
    *
-   * @param runCommand - The command string to check
+   * @param runCommand - The run command to check
    * @returns True if an adapter is registered for this command
    */
   has(runCommand) {
@@ -603,7 +615,7 @@ var AgentAdapterRegistry = class {
   /**
    * Get all supported commands.
    *
-   * @returns Array of all registered command strings
+   * @returns Array of all registered run commands
    */
   getSupportedCommands() {
     return Array.from(this.adapters.keys());
@@ -653,6 +665,9 @@ function getAdapter(runCommand) {
   return adapter;
 }
 
+// src/run-scenario/agents/claude-code/claude-code-adapter.ts
+var import_evalforge_types4 = require("@wix/evalforge-types");
+
 // src/run-scenario/agents/claude-code/execute.ts
 var import_evalforge_types3 = require("@wix/evalforge-types");
 var import_crypto = require("crypto");
@@ -1669,7 +1684,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
 var ClaudeCodeAdapter = class {
   id = "claude-code";
   name = "Claude Code";
-  supportedCommands = ["claude"];
+  supportedCommands = [import_evalforge_types4.AgentRunCommand.CLAUDE];
   /**
    * Execute a skill using the Claude Code SDK.
    *
@@ -2450,7 +2465,8 @@ function extractTemplateFiles(before, after) {
 }
 
 // src/run-scenario/run-agent-with-context.ts
-var DEFAULT_AGENT_COMMAND = "claude";
+var import_evalforge_types5 = require("@wix/evalforge-types");
+var DEFAULT_AGENT_COMMAND = import_evalforge_types5.AgentRunCommand.CLAUDE;
 async function runAgentWithContext(config, evalRunId2, scenario, evalData, workDir) {
   const skillsGroupId = evalData.evalRun.skillsGroupId;
   if (!skillsGroupId) {
@@ -2534,10 +2550,11 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     fileDiffs: partialResult.fileDiffs?.map((d) => ({
       path: d.path,
       status: templateFilesMap.get(d.path)
-    }))
+    })),
+    durationMs: partialResult.duration
   };
   const { "x-wix-ai-gateway-stream": _stream, ...judgeHeaders } = config.aiGatewayHeaders;
-  const defaultJudgeModel = import_evalforge_types4.DEFAULT_JUDGE_MODEL;
+  const defaultJudgeModel = import_evalforge_types6.DEFAULT_JUDGE_MODEL;
   const assertionContext = {
     workDir,
     defaultJudgeModel,
@@ -2552,10 +2569,10 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     assertionContext
   ) : [];
   const passed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types4.AssertionResultStatus.PASSED
+    (r) => r.status === import_evalforge_types6.AssertionResultStatus.PASSED
   ).length;
   const failed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types4.AssertionResultStatus.FAILED
+    (r) => r.status === import_evalforge_types6.AssertionResultStatus.FAILED
   ).length;
   const total = assertionResults.length;
   const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
@@ -2569,7 +2586,7 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
 }
 
 // src/error-reporter.ts
-var import_evalforge_types5 = require("@wix/evalforge-types");
+var import_evalforge_types7 = require("@wix/evalforge-types");
 function formatError(error, phase, context) {
   const timestamp = (/* @__PURE__ */ new Date()).toISOString();
   if (error instanceof Error) {
@@ -2818,7 +2835,7 @@ async function runEvaluation(projectId2, evalRunId2) {
   };
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {
-      status: import_evalforge_types6.EvalStatus.COMPLETED,
+      status: import_evalforge_types8.EvalStatus.COMPLETED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString()
     });
   } catch (updateErr) {
@@ -2859,7 +2876,7 @@ runEvaluation(projectId, evalRunId).then(() => {
       authToken: config.authToken
     });
     await api.updateEvalRun(projectId, evalRunId, {
-      status: import_evalforge_types6.EvalStatus.FAILED,
+      status: import_evalforge_types8.EvalStatus.FAILED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       jobError,
       jobStatus: "FAILED"
@@ -2882,7 +2899,7 @@ runEvaluation(projectId, evalRunId).then(() => {
           authToken
         });
         await api.updateEvalRun(projectId, evalRunId, {
-          status: import_evalforge_types6.EvalStatus.FAILED,
+          status: import_evalforge_types8.EvalStatus.FAILED,
           completedAt: (/* @__PURE__ */ new Date()).toISOString(),
           jobError: `Config load failed, then: ${jobError}`,
           jobStatus: "FAILED"