@wix/evalforge-evaluator 0.58.0 → 0.60.0

package/build/index.js

@@ -194,6 +194,16 @@ function createApiClient(serverUrl, options = "") {
 
 // src/fetch-evaluation-data.ts
 var import_evalforge_types = require("@wix/evalforge-types");
+function parseSkillNamesFromParams(value) {
+  if (typeof value !== "string") {
+    return [];
+  }
+  const parsed = JSON.parse(value);
+  if (Array.isArray(parsed)) {
+    return parsed.map(String);
+  }
+  return [];
+}
 function applyParamsToAssertion(assertion, params) {
   if (!params || Object.keys(params).length === 0) {
     return assertion;
@@ -215,6 +225,12 @@ function applyParamsToAssertion(assertion, params) {
     }
     return { ...assertion, prompt, systemPrompt };
   }
+  if (assertion.type === "skill_was_called" && params.skillNames !== void 0) {
+    return {
+      ...assertion,
+      skillNames: parseSkillNamesFromParams(params.skillNames)
+    };
+  }
   return { ...assertion, ...params };
 }
 function resolveSystemAssertion(assertionId, params) {
@@ -224,7 +240,7 @@ function resolveSystemAssertion(assertionId, params) {
     case "skill_was_called":
       baseAssertion = {
         type: "skill_was_called",
-        skillName: params?.skillName ?? ""
+        skillNames: parseSkillNamesFromParams(params?.skillNames)
       };
       break;
     case "build_passed":
@@ -249,38 +265,15 @@ function resolveSystemAssertion(assertionId, params) {
 }
 function customAssertionToAssertion(ca, params) {
   const config = ca.config;
-  let baseAssertion;
-  switch (ca.type) {
-    case "skill_was_called":
-      baseAssertion = {
-        type: "skill_was_called",
-        skillName: config?.skillName ?? ""
-      };
-      break;
-    case "build_passed":
-      baseAssertion = {
-        type: "build_passed",
-        command: config?.command,
-        expectedExitCode: config?.expectedExitCode
-      };
-      break;
-    case "llm_judge":
-      baseAssertion = {
-        type: "llm_judge",
-        prompt: config?.prompt ?? "",
-        systemPrompt: config?.systemPrompt,
-        minScore: config?.minScore,
-        model: config?.model,
-        maxTokens: config?.maxTokens,
-        temperature: config?.temperature
-      };
-      break;
-    default:
-      baseAssertion = {
-        type: "llm_judge",
-        prompt: ""
-      };
-  }
+  const baseAssertion = {
+    type: "llm_judge",
+    prompt: config?.prompt ?? "",
+    systemPrompt: config?.systemPrompt,
+    minScore: config?.minScore,
+    model: config?.model,
+    maxTokens: config?.maxTokens,
+    temperature: config?.temperature
+  };
   return applyParamsToAssertion(baseAssertion, params);
 }
 async function fetchEvaluationData(api, projectId2, evalRunId2) {
@@ -6376,7 +6369,15 @@ async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
   const mcpServers = {};
   for (const mcp of mcps) {
-    mcpServers[mcp.name] = mcp.config;
+    const config = mcp.config;
+    for (const [key, value] of Object.entries(config)) {
+      if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        throw new Error(
+          `MCP "${mcp.name}" has invalid config: value for key "${key}" must be an object (got ${typeof value}). Config must use keyed format, e.g. { "server-name": { "command": "npx", ... } }`
+        );
+      }
+      mcpServers[key] = value;
+    }
   }
   const content = JSON.stringify(
     { [import_evalforge_types2.MCP_SERVERS_JSON_KEY]: mcpServers },
@@ -6732,9 +6733,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
     })
   );
   let messageCount = 0;
-  const canUseTool = async () => {
-    return { behavior: "allow" };
-  };
+  const canUseTool = async (_toolName, input) => ({ behavior: "allow", updatedInput: input });
   const baseAllowedTools = [
     "Skill",
     "Read",