@wix/evalforge-evaluator 0.143.0 → 0.145.0

package/build/index.mjs

@@ -132,37 +132,23 @@ function createApiClient(serverUrl, options = "") {
     getScenario(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/test-scenarios/${id}`);
     },
-    getSkill(projectId2, id) {
-      return fetchJson(`/projects/${projectId2}/skills/${id}`);
-    },
-    getSkillVersion(projectId2, skillId, versionId) {
-      return fetchJson(
-        `/projects/${projectId2}/skills/${skillId}/versions/${versionId}`
-      );
-    },
-    getLatestSkillVersion(projectId2, skillId) {
-      return fetchJson(
-        `/projects/${projectId2}/skills/${skillId}/versions/latest`
-      );
-    },
     getAgent(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/agents/${id}`);
     },
     getTemplate(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/templates/${id}`);
     },
-    getMcp(projectId2, id) {
-      return fetchJson(`/projects/${projectId2}/mcps/${id}`);
-    },
-    getSubAgent(projectId2, id) {
-      return fetchJson(`/projects/${projectId2}/sub-agents/${id}`);
-    },
-    getRule(projectId2, id) {
-      return fetchJson(`/projects/${projectId2}/rules/${id}`);
-    },
     getPreset(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/presets/${id}`);
     },
+    getCapability(projectId2, id) {
+      return fetchJson(`/projects/${projectId2}/capabilities/${id}`);
+    },
+    getCapabilityVersion(projectId2, capabilityId, versionId) {
+      return fetchJson(
+        `/projects/${projectId2}/capabilities/${capabilityId}/versions/${versionId}`
+      );
+    },
     addResult(projectId2, evalRunId2, result) {
       return postJson(
         `/projects/${projectId2}/eval-runs/${evalRunId2}/results`,
@@ -184,7 +170,11 @@ function createApiClient(serverUrl, options = "") {
 import {
   isSystemAssertionId,
   SYSTEM_ASSERTIONS,
-  isAllowedBuildCommandString
+  isAllowedBuildCommandString,
+  capabilityToSkillWithLatestVersion,
+  capabilityToSubAgent,
+  capabilityToRule,
+  capabilityToMcp
 } from "@wix/evalforge-types";
 
 // src/resolve-placeholders.ts
@@ -326,55 +316,50 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
     agent = await api.getAgent(projectId2, evalRun.agentId);
   }
   let skills = [];
-  const resolvedSkillIds = evalRun.skillIds ?? [];
-  if (resolvedSkillIds.length > 0) {
-    const fetchResults = await Promise.allSettled(
-      resolvedSkillIds.map((id) => api.getSkill(projectId2, id))
-    );
-    skills = fetchResults.filter(
-      (r) => r.status === "fulfilled"
-    ).map((r) => r.value).filter((s) => !s.deleted);
-    if (evalRun.skillVersions && Object.keys(evalRun.skillVersions).length > 0) {
-      skills = await Promise.all(
-        skills.map(async (skill) => {
-          const versionId = evalRun.skillVersions?.[skill.id];
-          if (versionId) {
-            const version = await api.getSkillVersion(
-              projectId2,
-              skill.id,
-              versionId
-            );
-            return { ...skill, latestVersion: version };
-          }
-          return skill;
-        })
-      );
-    }
-    skills = skills.map((skill) => {
-      const hasPinnedVersion = evalRun.skillVersions?.[skill.id];
-      if (!hasPinnedVersion && skill.source) {
-        return { ...skill, latestVersion: void 0 };
-      }
-      return skill;
-    });
-  }
   let mcps = [];
-  if (evalRun.mcpIds && evalRun.mcpIds.length > 0) {
-    mcps = await Promise.all(
-      evalRun.mcpIds.map((id) => api.getMcp(projectId2, id))
-    );
-  }
   let subAgents = [];
-  if (evalRun.subAgentIds && evalRun.subAgentIds.length > 0) {
-    subAgents = await Promise.all(
-      evalRun.subAgentIds.map((id) => api.getSubAgent(projectId2, id))
-    );
-  }
   let rules = [];
-  if (evalRun.ruleIds && evalRun.ruleIds.length > 0) {
-    rules = await Promise.all(
-      evalRun.ruleIds.map((id) => api.getRule(projectId2, id))
+  if (evalRun.capabilityIds && evalRun.capabilityIds.length > 0) {
+    const fetchResults = await Promise.allSettled(
+      evalRun.capabilityIds.map((id) => api.getCapability(projectId2, id))
+    );
+    const capabilities = fetchResults.filter(
+      (r) => r.status === "fulfilled"
+    ).map((r) => r.value).filter((c) => !c.deleted);
+    const pinned = evalRun.capabilityVersions ?? {};
+    const withVersions = await Promise.all(
+      capabilities.map(async (cap) => {
+        const versionId = pinned[cap.id];
+        if (versionId) {
+          const version = await api.getCapabilityVersion(
+            projectId2,
+            cap.id,
+            versionId
+          );
+          return { ...cap, latestVersion: version };
+        }
+        if (cap.source) {
+          return { ...cap, latestVersion: void 0 };
+        }
+        return cap;
+      })
     );
+    for (const cap of withVersions) {
+      switch (cap.capabilityType) {
+        case "SKILL":
+          skills.push(capabilityToSkillWithLatestVersion(cap));
+          break;
+        case "SUB_AGENT":
+          subAgents.push(capabilityToSubAgent(cap));
+          break;
+        case "RULE":
+          rules.push(capabilityToRule(cap));
+          break;
+        case "MCP":
+          mcps.push(capabilityToMcp(cap));
+          break;
+      }
+    }
   }
   const templateIds = [
     ...new Set(
@@ -4987,7 +4972,7 @@ async function runEvaluation(projectId2, evalRunId2) {
     hasAgent: !!agent,
     agentId: evalData.evalRun.agentId,
     presetId: evalData.evalRun.presetId,
-    skillIds: evalData.evalRun.skillIds
+    capabilityIds: evalData.evalRun.capabilityIds
   };
   if (scenarioItems.length > 0 && !agent) {
     throw new Error(