@axiom-lattice/gateway 3.0.2 → 3.0.4

package/dist/index.mjs

@@ -6214,7 +6214,7 @@ var EvalRunner = class {
   getEventEmitter() {
     return this.eventEmitter;
   }
-  async startRun(tenantId, projectId, suiteIds) {
+  async startRun(tenantId, projectId, suiteIds, caseIds, runConfig) {
     const store = this.getEvalStore();
     const project = await store.getProjectById(tenantId, projectId);
     if (!project) throw new Error("Project not found");
@@ -6248,7 +6248,11 @@ var EvalRunner = class {
     const evalSuites = [];
     let totalCases = 0;
     for (const suite of suites) {
-      const cases = await store.getCasesBySuite(tenantId, suite.id);
+      let cases = await store.getCasesBySuite(tenantId, suite.id);
+      if (caseIds && caseIds.length > 0) {
+        cases = cases.filter((c) => caseIds.includes(c.id));
+        if (cases.length === 0) continue;
+      }
       totalCases += cases.length;
       evalSuites.push({
         suiteName: suite.name,
@@ -6268,6 +6272,9 @@ var EvalRunner = class {
         }))
       });
     }
+    if (caseIds && caseIds.length > 0 && evalSuites.length === 0) {
+      throw new Error("No cases match the requested caseIds filter");
+    }
     const judgeCfg = project.judgeModelConfig ?? {};
     const judgeModelKey = judgeCfg.modelKey || "";
     let resolvedJudgeModelKey;
@@ -6286,10 +6293,16 @@ var EvalRunner = class {
     }
     const runId = uuidv43();
     const concurrency = Math.max(1, Math.floor(project.concurrency) || 3);
-    await store.createRun(tenantId, projectId, runId, { totalCases, concurrency });
-    const tsc = project.targetServerConfig ?? {};
-    const workspaceId = tsc.workspace_id || "";
-    const projectIdFromConfig = tsc.project_id || "";
+    const isHoldout = suites.length > 0 && suites.every((s) => s.name.endsWith("-validation"));
+    const workspaceId = runConfig?.workspaceId || "";
+    const projectIdFromConfig = runConfig?.projectId || "";
+    await store.createRun(tenantId, projectId, runId, {
+      totalCases,
+      concurrency,
+      holdout: isHoldout,
+      envWorkspaceId: workspaceId || void 0,
+      envProjectId: projectIdFromConfig || void 0
+    });
     const projectConfig = {
       projectName: project.name,
       version: project.version,
@@ -6344,7 +6357,7 @@ var EvalRunner = class {
       try {
         const evalProject = new LatticeEvalProject(projectConfig, onCaseComplete);
         const calibration = await evalProject.calibrateJudge();
-        if (!calibration.ok) {
+        if (!calibration.ok && !calibration.bypassed) {
           await store.updateRunStatus(tenantId, runId, {
             status: "failed",
             error: `Judge calibration failed: ${calibration.reason}`,
@@ -6352,6 +6365,9 @@ var EvalRunner = class {
           });
           throw new Error(`Judge calibration failed: ${calibration.reason}`);
         }
+        if (calibration.bypassed) {
+          console.warn(`[eval-runner] Judge calibration bypassed \u2014 judge may be unreliable: ${calibration.reason}`);
+        }
         const { report } = await evalProject.runAllSuitesBatch(concurrency, abortController.signal);
         if (!abortController.signal.aborted) {
           const completedCount = stats.passed + stats.failed;
@@ -6401,6 +6417,12 @@ var EvalRunner = class {
     await store.updateRunStatus(ctx.tenantId, runId, { status: "aborted", completedAt: /* @__PURE__ */ new Date() });
     return true;
   }
+  async waitForRun(runId) {
+    const ctx = this.runs.get(runId);
+    if (!ctx) throw new Error("Run not found or no longer in memory");
+    const { report } = await ctx.promise;
+    return report;
+  }
   isRunning(runId) {
     return this.runs.has(runId);
   }
@@ -6740,11 +6762,17 @@ function registerEvalRoutes(app2) {
       const tenantId = getTenantId13(request);
       const { pid } = request.params;
       const body = request.body ?? {};
-      const runId = await evalRunner.startRun(tenantId, pid, body?.suiteIds);
+      const bodyWs = request.headers["x-workspace-id"] || "";
+      const bodyPj = request.headers["x-project-id"] || "";
+      const runConfig = {
+        workspaceId: body?.runConfig?.workspaceId ?? bodyWs,
+        projectId: body?.runConfig?.projectId ?? bodyPj
+      };
+      const runId = await evalRunner.startRun(tenantId, pid, body?.suiteIds, body?.caseIds, runConfig);
       reply.status(202).send({ success: true, message: "Run started", data: { run_id: runId } });
     } catch (err) {
       const msg = err.message;
-      const code = msg === "Project not found" ? 404 : msg.includes("already in progress") ? 409 : msg.includes("No suites match") ? 400 : 500;
+      const code = msg === "Project not found" ? 404 : msg.includes("already in progress") ? 409 : msg.includes("No suites match") || msg.includes("No cases match") ? 400 : 500;
       reply.status(code).send({ success: false, message: msg });
     }
   });