niceeval 0.13.1 → 0.13.2-canary.35

package/dist/runner/run.cjs

@@ -48,6 +48,40 @@ function feedbackIdentity(a) {
 function feedbackWho(a) {
     return (0, types_ts_1.runWho)({ agentName: a.run.agent.name, model: a.run.model, experimentId: a.run.experimentId });
 }
+const SHARED_BUILD_FAILURE_DIAGNOSTIC = "sandbox-build-failed";
+function sharedBuildFailureRoot(error) {
+    const codeMatch = /\bERR_[A-Z0-9_]+\b/u.exec(error.message);
+    const code = codeMatch?.[0] ?? error.code;
+    const focused = codeMatch === null
+        ? error.message.slice(-600)
+        : error.message.slice(codeMatch.index, codeMatch.index + 600);
+    return { code, message: focused.trim() || error.message.slice(-600).trim() || error.code };
+}
+function sharedBuildFailureDetail(a, error) {
+    const failureId = error.origin.scope === "run"
+        ? error.origin.timingNodeId
+        : `${error.code}:${a.evalDef.id}`;
+    const root = sharedBuildFailureRoot(error);
+    let message = root.message;
+    const encode = () => JSON.stringify({
+        schema: "niceeval.shared-build-failure/v1",
+        failureId,
+        evalId: a.evalDef.id,
+        attemptOrdinal: a.attempt,
+        phase: "sandbox.image.build",
+        errorCode: root.code,
+        message,
+        ...(root.code === "ERR_PNPM_IGNORED_BUILDS"
+            ? { remediation: "pnpm-allow-builds" }
+            : {}),
+    });
+    let encoded = encode();
+    while (new TextEncoder().encode(encoded).byteLength > 1000 && message.length > 0) {
+        message = message.slice(0, Math.floor(message.length * 0.75));
+        encoded = encode();
+    }
+    return encoded;
+}
 function attemptIdentityKey(experimentId, agentName, model, evalId) {
     return JSON.stringify([experimentId, agentName, model ?? null, evalId]);
 }
@@ -1809,6 +1843,15 @@ function runEvals(opts) {
                                 : undefined;
                 if (blockedError !== undefined)
                     cancelReuseAttempt(a, true);
+                if (buildFailure !== undefined) {
+                    recordExperimentDiagnostic({
+                        experimentId: a.run.experimentId,
+                        code: SHARED_BUILD_FAILURE_DIAGNOSTIC,
+                        level: "error",
+                        message: sharedBuildFailureDetail(a, buildFailure),
+                        phase: "sandbox.queue",
+                    });
+                }
                 // A scheduler-admitted Attempt owns a durable identity before any
                 // attempt-owned sandbox, Agent, or Eval work begins. Reservation
                 // failure is invocation-fatal at the Record publish gate.