@axiom-lattice/gateway 3.0.2 → 3.0.3

package/dist/index.js

@@ -8222,7 +8222,7 @@ var EvalRunner = class {
   getEventEmitter() {
     return this.eventEmitter;
   }
-  async startRun(tenantId, projectId, suiteIds) {
+  async startRun(tenantId, projectId, suiteIds, caseIds, runConfig) {
     const store2 = this.getEvalStore();
     const project = await store2.getProjectById(tenantId, projectId);
     if (!project) throw new Error("Project not found");
@@ -8256,7 +8256,11 @@ var EvalRunner = class {
     const evalSuites = [];
     let totalCases = 0;
     for (const suite of suites) {
-      const cases = await store2.getCasesBySuite(tenantId, suite.id);
+      let cases = await store2.getCasesBySuite(tenantId, suite.id);
+      if (caseIds && caseIds.length > 0) {
+        cases = cases.filter((c) => caseIds.includes(c.id));
+        if (cases.length === 0) continue;
+      }
       totalCases += cases.length;
       evalSuites.push({
         suiteName: suite.name,
@@ -8276,6 +8280,9 @@ var EvalRunner = class {
         }))
       });
     }
+    if (caseIds && caseIds.length > 0 && evalSuites.length === 0) {
+      throw new Error("No cases match the requested caseIds filter");
+    }
     const judgeCfg = project.judgeModelConfig ?? {};
     const judgeModelKey = judgeCfg.modelKey || "";
     let resolvedJudgeModelKey;
@@ -8294,10 +8301,16 @@ var EvalRunner = class {
     }
     const runId = (0, import_uuid4.v4)();
     const concurrency = Math.max(1, Math.floor(project.concurrency) || 3);
-    await store2.createRun(tenantId, projectId, runId, { totalCases, concurrency });
-    const tsc = project.targetServerConfig ?? {};
-    const workspaceId = tsc.workspace_id || "";
-    const projectIdFromConfig = tsc.project_id || "";
+    const isHoldout = suites.length > 0 && suites.every((s) => s.name.endsWith("-validation"));
+    const workspaceId = runConfig?.workspaceId || "";
+    const projectIdFromConfig = runConfig?.projectId || "";
+    await store2.createRun(tenantId, projectId, runId, {
+      totalCases,
+      concurrency,
+      holdout: isHoldout,
+      envWorkspaceId: workspaceId || void 0,
+      envProjectId: projectIdFromConfig || void 0
+    });
     const projectConfig = {
       projectName: project.name,
       version: project.version,
@@ -8352,7 +8365,7 @@ var EvalRunner = class {
       try {
         const evalProject = new import_core31.LatticeEvalProject(projectConfig, onCaseComplete);
         const calibration = await evalProject.calibrateJudge();
-        if (!calibration.ok) {
+        if (!calibration.ok && !calibration.bypassed) {
           await store2.updateRunStatus(tenantId, runId, {
             status: "failed",
             error: `Judge calibration failed: ${calibration.reason}`,
@@ -8360,6 +8373,9 @@ var EvalRunner = class {
           });
           throw new Error(`Judge calibration failed: ${calibration.reason}`);
         }
+        if (calibration.bypassed) {
+          console.warn(`[eval-runner] Judge calibration bypassed \u2014 judge may be unreliable: ${calibration.reason}`);
+        }
         const { report } = await evalProject.runAllSuitesBatch(concurrency, abortController.signal);
         if (!abortController.signal.aborted) {
           const completedCount = stats.passed + stats.failed;
@@ -8748,11 +8764,17 @@ function registerEvalRoutes(app2) {
       const tenantId = getTenantId14(request);
       const { pid } = request.params;
       const body = request.body ?? {};
-      const runId = await evalRunner.startRun(tenantId, pid, body?.suiteIds);
+      const bodyWs = request.headers["x-workspace-id"] || "";
+      const bodyPj = request.headers["x-project-id"] || "";
+      const runConfig = {
+        workspaceId: body?.runConfig?.workspaceId ?? bodyWs,
+        projectId: body?.runConfig?.projectId ?? bodyPj
+      };
+      const runId = await evalRunner.startRun(tenantId, pid, body?.suiteIds, body?.caseIds, runConfig);
       reply.status(202).send({ success: true, message: "Run started", data: { run_id: runId } });
     } catch (err) {
       const msg = err.message;
-      const code = msg === "Project not found" ? 404 : msg.includes("already in progress") ? 409 : msg.includes("No suites match") ? 400 : 500;
+      const code = msg === "Project not found" ? 404 : msg.includes("already in progress") ? 409 : msg.includes("No suites match") || msg.includes("No cases match") ? 400 : 500;
       reply.status(code).send({ success: false, message: msg });
     }
   });