@wix/evalforge-evaluator 0.221.0 → 0.223.0

package/build/index.js

@@ -6941,6 +6941,12 @@ function rethrowWithRequestId(err, action) {
   }
   throw err;
 }
+function requireEntity(entity, notFound) {
+  if (!entity) {
+    throw new Error(notFound);
+  }
+  return entity;
+}
 function resolveAmbassadorBaseUrl(serverUrl) {
   try {
     return new URL(serverUrl).origin;
@@ -6991,52 +6997,50 @@ function createApiClient(serverUrl, options = "") {
       const res = await httpClient.request(
         getEvalRun({ projectId: projectId2, evalRunId: id })
       );
-      const evalRun = res.data.evalRun;
-      if (!evalRun) {
-        throw new Error(`Eval run ${id} not found in project ${projectId2}`);
-      }
+      const evalRun = requireEntity(
+        res.data.evalRun,
+        `Eval run ${id} not found in project ${projectId2}`
+      );
       return evalRunFromProto(evalRun);
     },
     async getScenario(projectId2, id) {
       const res = await httpClient.request(
         getTestScenario({ projectId: projectId2, testScenarioId: id })
       );
-      const scenario = res.data.testScenario;
-      if (!scenario) {
-        throw new Error(
-          `Test scenario ${id} not found in project ${projectId2}`
-        );
-      }
+      const scenario = requireEntity(
+        res.data.testScenario,
+        `Test scenario ${id} not found in project ${projectId2}`
+      );
       return testScenarioFromProto(scenario);
     },
     async getAgent(projectId2, id) {
       const res = await httpClient.request(
         getAgent({ projectId: projectId2, agentId: id })
       );
-      const agent = res.data.agent;
-      if (!agent) {
-        throw new Error(`Agent ${id} not found in project ${projectId2}`);
-      }
+      const agent = requireEntity(
+        res.data.agent,
+        `Agent ${id} not found in project ${projectId2}`
+      );
       return agentFromProto(agent);
     },
     async getTemplate(projectId2, id) {
       const res = await httpClient.request(
         getTemplate({ projectId: projectId2, templateId: id })
       );
-      const template = res.data.template;
-      if (!template) {
-        throw new Error(`Template ${id} not found in project ${projectId2}`);
-      }
+      const template = requireEntity(
+        res.data.template,
+        `Template ${id} not found in project ${projectId2}`
+      );
       return templateFromProto(template);
     },
     async getPreset(projectId2, id) {
       const res = await httpClient.request(
         getPreset({ projectId: projectId2, presetId: id })
       );
-      const preset = res.data.preset;
-      if (!preset) {
-        throw new Error(`Preset ${id} not found in project ${projectId2}`);
-      }
+      const preset = requireEntity(
+        res.data.preset,
+        `Preset ${id} not found in project ${projectId2}`
+      );
       return presetFromProto(preset);
     },
     // The legacy REST endpoint enriched the capability with its latest version
@@ -7057,10 +7061,10 @@ function createApiClient(serverUrl, options = "") {
       if (capResult.status === "rejected") {
         throw capResult.reason;
       }
-      const capability = capResult.value.data.capability;
-      if (!capability) {
-        throw new Error(`Capability ${id} not found in project ${projectId2}`);
-      }
+      const capability = requireEntity(
+        capResult.value.data.capability,
+        `Capability ${id} not found in project ${projectId2}`
+      );
       let latestVersion;
       if (versionResult.status === "fulfilled" && versionResult.value.data.capabilityVersion) {
         latestVersion = capabilityVersionFromProto(
@@ -7083,12 +7087,10 @@ function createApiClient(serverUrl, options = "") {
           capabilityVersionId: versionId
         })
       );
-      const version = res.data.capabilityVersion;
-      if (!version) {
-        throw new Error(
-          `Capability version ${versionId} not found for capability ${capabilityId}`
-        );
-      }
+      const version = requireEntity(
+        res.data.capabilityVersion,
+        `Capability version ${versionId} not found for capability ${capabilityId}`
+      );
       return capabilityVersionFromProto(version, projectId2);
     },
     async addResult(projectId2, evalRunId2, result) {
@@ -7415,7 +7417,6 @@ async function writeFilesToDirectory(targetDir, files) {
 
 // src/run-scenario/install-dependencies.ts
 var import_fs = require("fs");
-var import_crypto = require("crypto");
 var import_path2 = __toESM(require("path"));
 var import_child_process = require("child_process");
 var INSTALL_TIMEOUT_MS = 18e4;
@@ -7514,18 +7515,16 @@ function detectPackageManager(workDir) {
   if ((0, import_fs.existsSync)(import_path2.default.join(workDir, "pnpm-lock.yaml"))) {
     return {
       cmd: "pnpm",
-      args: ["install", "--frozen-lockfile"],
-      cacheSourceFile: "pnpm-lock.yaml"
+      args: ["install", "--frozen-lockfile"]
     };
   }
   if ((0, import_fs.existsSync)(import_path2.default.join(workDir, "package-lock.json"))) {
-    return { cmd: "npm", args: ["ci"], cacheSourceFile: "package-lock.json" };
+    return { cmd: "npm", args: ["ci"] };
   }
   if ((0, import_fs.existsSync)(import_path2.default.join(workDir, "yarn.lock"))) {
     return {
       cmd: "yarn",
-      args: ["install", "--frozen-lockfile"],
-      cacheSourceFile: "yarn.lock"
+      args: ["install", "--frozen-lockfile"]
     };
   }
   return {
@@ -7536,17 +7535,9 @@ function detectPackageManager(workDir) {
       "--prefer-offline",
       "--no-fund",
       "--no-audit"
-    ],
-    cacheSourceFile: "package.json"
+    ]
   };
 }
-function cloneDirectory(src, dest) {
-  if (process.platform === "darwin") {
-    (0, import_child_process.execFileSync)("cp", ["-rc", src, dest]);
-  } else {
-    (0, import_fs.cpSync)(src, dest, { recursive: true });
-  }
-}
 async function runInstall(exec, pm, workDir, onProgress) {
   reportRegistry(workDir, onProgress);
   onProgress(`[diag] npm install starting: ${pm.cmd} ${pm.args.join(" ")}`);
@@ -7577,64 +7568,13 @@ async function runInstall(exec, pm, workDir, onProgress) {
     clearInterval(heartbeat);
   }
 }
-async function installWithCache(workDir, exec, cacheBase, pm, onProgress) {
-  const sourceContent = (0, import_fs.readFileSync)(
-    import_path2.default.join(workDir, pm.cacheSourceFile),
-    "utf-8"
-  );
-  const cacheKey = (0, import_crypto.createHash)("sha256").update(sourceContent).digest("hex").slice(0, 16);
-  const cachedNodeModules = import_path2.default.join(cacheBase, cacheKey, "node_modules");
-  const targetNodeModules = import_path2.default.join(workDir, "node_modules");
-  const cacheDir = import_path2.default.dirname(cachedNodeModules);
-  const cachedYarnLock = import_path2.default.join(cacheDir, "yarn.lock");
-  if ((0, import_fs.existsSync)(cachedNodeModules)) {
-    console.log(
-      `[environment] Restoring node_modules from cache (key: ${cacheKey})`
-    );
-    onProgress(`[diag] node_modules cache HIT (key: ${cacheKey}) \u2014 restoring`);
-    if (!(0, import_fs.existsSync)(targetNodeModules)) {
-      cloneDirectory(cachedNodeModules, targetNodeModules);
-    }
-    if ((0, import_fs.existsSync)(cachedYarnLock)) {
-      (0, import_fs.copyFileSync)(cachedYarnLock, import_path2.default.join(workDir, "yarn.lock"));
-    }
-    onProgress("[diag] node_modules cache restore complete");
-    return;
-  }
-  onProgress(`[diag] node_modules cache MISS (key: ${cacheKey})`);
-  const ok = await runInstall(exec, pm, workDir, onProgress);
-  if (!ok) {
-    return;
-  }
-  console.log(
-    "[environment] Dependency installation complete \u2014 saving to cache"
-  );
-  try {
-    (0, import_fs.mkdirSync)(cacheDir, { recursive: true });
-    const yarnLockPath = import_path2.default.join(workDir, "yarn.lock");
-    if ((0, import_fs.existsSync)(yarnLockPath)) {
-      (0, import_fs.copyFileSync)(yarnLockPath, cachedYarnLock);
-    }
-    cloneDirectory(targetNodeModules, cachedNodeModules);
-  } catch (err) {
-    console.error(
-      "[environment] Failed to save to cache (installation still succeeded):",
-      err instanceof Error ? err.message : String(err)
-    );
-  }
-}
 async function installDependencies(workDir, onProgress, options = {}) {
   if (!(0, import_fs.existsSync)(import_path2.default.join(workDir, "package.json"))) {
     return;
   }
   const exec = options.exec ?? defaultExec;
-  const cacheBase = options.cacheBase;
   onProgress("Installing dependencies...");
   const pm = detectPackageManager(workDir);
-  if (cacheBase) {
-    await installWithCache(workDir, exec, cacheBase, pm, onProgress);
-    return;
-  }
   await runInstall(exec, pm, workDir, onProgress);
 }
 
@@ -7779,7 +7719,6 @@ function writeWixEnvFile(workDir) {
 async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId, onProgress, options = {}) {
   const template = options.template;
   const baseDir = config.evaluationsDir ?? import_path3.default.join((0, import_os.tmpdir)(), "evalforge-evaluations");
-  const nodeModulesCacheDir = import_path3.default.join(baseDir, "_node_modules_cache");
   if (template) {
     if (!config.evaluationsDir) {
       console.warn(
@@ -7798,7 +7737,6 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
     writeWixEnvFile(workDir2);
     onProgress("[diag] entering installDependencies");
     await installDependencies(workDir2, onProgress, {
-      cacheBase: nodeModulesCacheDir,
       exec: createLoggingInstallExec(onProgress)
     });
     onProgress("[diag] installDependencies returned");
@@ -7822,7 +7760,7 @@ function emitTraceEvent(event, pushEvent) {
 }
 
 // src/run-scenario/run-agent-with-context.ts
-var import_crypto5 = require("crypto");
+var import_crypto4 = require("crypto");
 
 // src/run-scenario/agents/registry.ts
 var AgentAdapterRegistry = class {
@@ -7980,7 +7918,7 @@ function resolveTimeoutMs(maxTurns, maxDurationMs) {
 }
 
 // src/run-scenario/agents/claude-code/execute.ts
-var import_crypto2 = require("crypto");
+var import_crypto = require("crypto");
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
 var import_promises5 = require("fs/promises");
@@ -9177,7 +9115,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     const totalSubSteps = thinkingSubSteps + toolSubSteps + textSubSteps || 1;
     if (hasThinking && (hasText || toolCallCount > 0)) {
       subSteps.push({
-        id: (0, import_crypto2.randomUUID)(),
+        id: (0, import_crypto.randomUUID)(),
         stepNumber: 0,
         // renumbered below
         turnIndex,
@@ -9209,7 +9147,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         const toolSuccess = !tc.isError;
         const toolError = tc.isError ? tc.errorContent ?? "Tool call failed" : void 0;
         subSteps.push({
-          id: (0, import_crypto2.randomUUID)(),
+          id: (0, import_crypto.randomUUID)(),
           stepNumber: 0,
           turnIndex,
           type: import_evalforge_types4.LLMStepType.TOOL_USE,
@@ -9239,7 +9177,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     }
     if (hasText && toolCallCount > 0) {
       subSteps.push({
-        id: (0, import_crypto2.randomUUID)(),
+        id: (0, import_crypto.randomUUID)(),
         stepNumber: 0,
         turnIndex,
         type: import_evalforge_types4.LLMStepType.COMPLETION,
@@ -9261,7 +9199,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     if (subSteps.length === 0) {
       const stepType = hasThinking && !hasText ? import_evalforge_types4.LLMStepType.THINKING : import_evalforge_types4.LLMStepType.COMPLETION;
       subSteps.push({
-        id: (0, import_crypto2.randomUUID)(),
+        id: (0, import_crypto.randomUUID)(),
         stepNumber: 0,
         turnIndex,
         type: stepType,
@@ -9319,7 +9257,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     stepTypeBreakdown
   };
   return {
-    id: (0, import_crypto2.randomUUID)(),
+    id: (0, import_crypto.randomUUID)(),
     steps: traceSteps,
     summary
   };
@@ -9686,7 +9624,7 @@ async function buildOpenCodeEnv(options) {
 
 // src/run-scenario/agents/opencode/build-trace.ts
 var import_evalforge_types7 = require("@wix/evalforge-types");
-var import_crypto3 = require("crypto");
+var import_crypto2 = require("crypto");
 function toCanonicalModelId(modelId) {
   const slashIndex = modelId.indexOf("/");
   return slashIndex > 0 ? modelId.slice(slashIndex + 1) : modelId;
@@ -9798,7 +9736,7 @@ function buildTurnSteps(turn, turnIndex, ctx) {
   const totalSubSteps = thinkingSubSteps + toolSubSteps + textSubSteps || 1;
   if (hasThinking && (hasText || toolCallCount > 0)) {
     subSteps.push({
-      id: (0, import_crypto3.randomUUID)(),
+      id: (0, import_crypto2.randomUUID)(),
       stepNumber: 0,
       turnIndex,
       type: import_evalforge_types7.LLMStepType.THINKING,
@@ -9825,7 +9763,7 @@ function buildTurnSteps(turn, turnIndex, ctx) {
       const toolFraction = toolBudgetSteps > 0 ? 1 / toolBudgetSteps : 1;
       const remainingFraction = (totalSubSteps - thinkingSubSteps) / totalSubSteps;
       subSteps.push({
-        id: (0, import_crypto3.randomUUID)(),
+        id: (0, import_crypto2.randomUUID)(),
         stepNumber: 0,
         turnIndex,
         type: import_evalforge_types7.LLMStepType.TOOL_USE,
@@ -9855,7 +9793,7 @@ function buildTurnSteps(turn, turnIndex, ctx) {
   }
   if (hasText && toolCallCount > 0) {
     subSteps.push({
-      id: (0, import_crypto3.randomUUID)(),
+      id: (0, import_crypto2.randomUUID)(),
       stepNumber: 0,
       turnIndex,
       type: import_evalforge_types7.LLMStepType.COMPLETION,
@@ -9877,7 +9815,7 @@ function buildTurnSteps(turn, turnIndex, ctx) {
   if (subSteps.length === 0) {
     const stepType = hasThinking && !hasText ? import_evalforge_types7.LLMStepType.THINKING : import_evalforge_types7.LLMStepType.COMPLETION;
     subSteps.push({
-      id: (0, import_crypto3.randomUUID)(),
+      id: (0, import_crypto2.randomUUID)(),
       stepNumber: 0,
       turnIndex,
       type: stepType,
@@ -9967,7 +9905,7 @@ function buildLLMTrace(timestampedEvents, totalDurationMs, model, provider, exec
     totalDurationMs,
     canonicalModel
   );
-  return { id: (0, import_crypto3.randomUUID)(), steps: allSteps, summary };
+  return { id: (0, import_crypto2.randomUUID)(), steps: allSteps, summary };
 }
 
 // src/run-scenario/agents/opencode/gateway-cost-interceptor.ts
@@ -10836,7 +10774,7 @@ var import_anthropic = require("@ai-sdk/anthropic");
 var import_google = require("@ai-sdk/google");
 var import_openai = require("@ai-sdk/openai");
 var import_evalforge_types11 = require("@wix/evalforge-types");
-var import_crypto4 = require("crypto");
+var import_crypto3 = require("crypto");
 
 // src/run-scenario/agents/simple-agent/mcp-tools.ts
 var import_mcp = require("@ai-sdk/mcp");
@@ -11473,7 +11411,7 @@ function buildLLMTrace2(steps, totalDurationMs, modelId, provider, executionStar
     const costUsd = calculateStepCost(step, modelId, provider, tokenUsage);
     const toolResultError = findToolResultError(step);
     return {
-      id: (0, import_crypto4.randomUUID)(),
+      id: (0, import_crypto3.randomUUID)(),
       stepNumber: i + 1,
       turnIndex: i,
       type: step.toolCalls.length > 0 ? import_evalforge_types11.LLMStepType.TOOL_USE : import_evalforge_types11.LLMStepType.COMPLETION,
@@ -11500,7 +11438,7 @@ function buildLLMTrace2(steps, totalDurationMs, modelId, provider, executionStar
     { prompt: 0, completion: 0, total: 0 }
   );
   return {
-    id: (0, import_crypto4.randomUUID)(),
+    id: (0, import_crypto3.randomUUID)(),
     steps: traceSteps,
     summary: {
       totalSteps: traceSteps.length,
@@ -12383,7 +12321,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     }
   } : agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
   return {
-    id: (0, import_crypto5.randomUUID)(),
+    id: (0, import_crypto4.randomUUID)(),
     targetId,
     targetName,
     scenarioId: scenario.id,
@@ -12529,7 +12467,7 @@ Site ID: ${provisionedSite.id}` : void 0;
 }
 
 // src/evaluation-loop.ts
-var import_crypto6 = require("crypto");
+var import_crypto5 = require("crypto");
 async function runEvaluationLoop(scenarioItems, evalData, callbacks) {
   const runsPerScenario = evalData.evalRun.runsPerScenario ?? 1;
   let completedExecutions = 0;
@@ -12555,7 +12493,7 @@ async function runEvaluationLoop(scenarioItems, evalData, callbacks) {
           `[Evaluator] Scenario iteration failed, recording as error result: "${scenario.name}"${iterLabel} \u2014 ${errorMsg}`
         );
         const errorResult = {
-          id: (0, import_crypto6.randomUUID)(),
+          id: (0, import_crypto5.randomUUID)(),
           targetId,
           targetName,
           scenarioId: scenario.id,