@wix/evalforge-evaluator 0.143.0 → 0.144.0

package/build/index.js

@@ -186,6 +186,14 @@ function createApiClient(serverUrl, options = "") {
     getPreset(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/presets/${id}`);
     },
+    getCapability(projectId2, id) {
+      return fetchJson(`/projects/${projectId2}/capabilities/${id}`);
+    },
+    getCapabilityVersion(projectId2, capabilityId, versionId) {
+      return fetchJson(
+        `/projects/${projectId2}/capabilities/${capabilityId}/versions/${versionId}`
+      );
+    },
     addResult(projectId2, evalRunId2, result) {
       return postJson(
         `/projects/${projectId2}/eval-runs/${evalRunId2}/results`,
@@ -345,55 +353,98 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
     agent = await api.getAgent(projectId2, evalRun.agentId);
   }
   let skills = [];
-  const resolvedSkillIds = evalRun.skillIds ?? [];
-  if (resolvedSkillIds.length > 0) {
+  let mcps = [];
+  let subAgents = [];
+  let rules = [];
+  if (evalRun.capabilityIds && evalRun.capabilityIds.length > 0) {
     const fetchResults = await Promise.allSettled(
-      resolvedSkillIds.map((id) => api.getSkill(projectId2, id))
+      evalRun.capabilityIds.map((id) => api.getCapability(projectId2, id))
     );
-    skills = fetchResults.filter(
+    const capabilities = fetchResults.filter(
       (r) => r.status === "fulfilled"
-    ).map((r) => r.value).filter((s) => !s.deleted);
-    if (evalRun.skillVersions && Object.keys(evalRun.skillVersions).length > 0) {
-      skills = await Promise.all(
-        skills.map(async (skill) => {
-          const versionId = evalRun.skillVersions?.[skill.id];
-          if (versionId) {
-            const version = await api.getSkillVersion(
-              projectId2,
-              skill.id,
-              versionId
-            );
-            return { ...skill, latestVersion: version };
-          }
-          return skill;
-        })
-      );
+    ).map((r) => r.value).filter((c) => !c.deleted);
+    const pinned = evalRun.capabilityVersions ?? {};
+    const withVersions = await Promise.all(
+      capabilities.map(async (cap) => {
+        const versionId = pinned[cap.id];
+        if (versionId) {
+          const version = await api.getCapabilityVersion(
+            projectId2,
+            cap.id,
+            versionId
+          );
+          return { ...cap, latestVersion: version };
+        }
+        if (cap.source) {
+          return { ...cap, latestVersion: void 0 };
+        }
+        return cap;
+      })
+    );
+    for (const cap of withVersions) {
+      switch (cap.capabilityType) {
+        case "SKILL":
+          skills.push((0, import_evalforge_types.capabilityToSkillWithLatestVersion)(cap));
+          break;
+        case "SUB_AGENT":
+          subAgents.push((0, import_evalforge_types.capabilityToSubAgent)(cap));
+          break;
+        case "RULE":
+          rules.push((0, import_evalforge_types.capabilityToRule)(cap));
+          break;
+        case "MCP":
+          mcps.push((0, import_evalforge_types.capabilityToMcp)(cap));
+          break;
+      }
     }
-    skills = skills.map((skill) => {
-      const hasPinnedVersion = evalRun.skillVersions?.[skill.id];
-      if (!hasPinnedVersion && skill.source) {
-        return { ...skill, latestVersion: void 0 };
+  } else {
+    const resolvedSkillIds = evalRun.skillIds ?? [];
+    if (resolvedSkillIds.length > 0) {
+      const fetchResults = await Promise.allSettled(
+        resolvedSkillIds.map((id) => api.getSkill(projectId2, id))
+      );
+      skills = fetchResults.filter(
+        (r) => r.status === "fulfilled"
+      ).map((r) => r.value).filter((s) => !s.deleted);
+      if (evalRun.skillVersions && Object.keys(evalRun.skillVersions).length > 0) {
+        skills = await Promise.all(
+          skills.map(async (skill) => {
+            const versionId = evalRun.skillVersions?.[skill.id];
+            if (versionId) {
+              const version = await api.getSkillVersion(
+                projectId2,
+                skill.id,
+                versionId
+              );
+              return { ...skill, latestVersion: version };
+            }
+            return skill;
+          })
+        );
       }
-      return skill;
-    });
-  }
-  let mcps = [];
-  if (evalRun.mcpIds && evalRun.mcpIds.length > 0) {
-    mcps = await Promise.all(
-      evalRun.mcpIds.map((id) => api.getMcp(projectId2, id))
-    );
-  }
-  let subAgents = [];
-  if (evalRun.subAgentIds && evalRun.subAgentIds.length > 0) {
-    subAgents = await Promise.all(
-      evalRun.subAgentIds.map((id) => api.getSubAgent(projectId2, id))
-    );
-  }
-  let rules = [];
-  if (evalRun.ruleIds && evalRun.ruleIds.length > 0) {
-    rules = await Promise.all(
-      evalRun.ruleIds.map((id) => api.getRule(projectId2, id))
-    );
+      skills = skills.map((skill) => {
+        const hasPinnedVersion = evalRun.skillVersions?.[skill.id];
+        if (!hasPinnedVersion && skill.source) {
+          return { ...skill, latestVersion: void 0 };
+        }
+        return skill;
+      });
+    }
+    if (evalRun.mcpIds && evalRun.mcpIds.length > 0) {
+      mcps = await Promise.all(
+        evalRun.mcpIds.map((id) => api.getMcp(projectId2, id))
+      );
+    }
+    if (evalRun.subAgentIds && evalRun.subAgentIds.length > 0) {
+      subAgents = await Promise.all(
+        evalRun.subAgentIds.map((id) => api.getSubAgent(projectId2, id))
+      );
+    }
+    if (evalRun.ruleIds && evalRun.ruleIds.length > 0) {
+      rules = await Promise.all(
+        evalRun.ruleIds.map((id) => api.getRule(projectId2, id))
+      );
+    }
   }
   const templateIds = [
     ...new Set(