@tangle-network/agent-eval 0.127.0 → 0.128.0

package/dist/index.js

@@ -9,11 +9,11 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-FA3MFYNA.js";
+} from "./chunk-NACAGYSY.js";
 import {
   mintRolloutRows,
   rolloutReward
-} from "./chunk-UTA2TGC4.js";
+} from "./chunk-IHQDPH7D.js";
 import {
   SUPERVISOR_RUN_SCHEMA,
   analyzeSupervisorRun,
@@ -45,7 +45,7 @@ import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
   deterministicSplit
-} from "./chunk-ARCFIY7U.js";
+} from "./chunk-XPRT64IE.js";
 import {
   DEFAULT_RULES,
   classifyFailure,
@@ -84,7 +84,7 @@ import {
   harnessAxisOf,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-PRB53E3J.js";
+} from "./chunk-UB2LOJ6Q.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -118,7 +118,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-65QETEKG.js";
+} from "./chunk-NKAGIDE2.js";
 import {
   BackendIntegrityError,
   assertRealAgentReceipts,
@@ -130,7 +130,7 @@ import {
   inMemoryVerdictCache,
   summarizeAgentReceiptIntegrity,
   summarizeBackendIntegrity
-} from "./chunk-WED5KLZQ.js";
+} from "./chunk-EZJEIH2R.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   FindingsStore,
@@ -189,19 +189,19 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-OD5NQ2CS.js";
+} from "./chunk-BYT7ELPS.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-Y5CYOL42.js";
+} from "./chunk-XDWDC2MP.js";
 import {
   runEvalCampaign
-} from "./chunk-CLELFRQG.js";
-import "./chunk-424XHVKV.js";
-import "./chunk-ZPUEEM5U.js";
+} from "./chunk-TBL77AUT.js";
+import "./chunk-NYLOYM6N.js";
+import "./chunk-2MKQIFS4.js";
 import {
   LlmCallError,
   LlmClient,
@@ -228,7 +228,7 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-IMGJZBFG.js";
+} from "./chunk-VLOATJQ2.js";
 import {
   comparePairedArms,
   pairArms,
@@ -326,8 +326,8 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-X6VOPMLD.js";
-import "./chunk-IN455XKS.js";
+} from "./chunk-EOSZT7PL.js";
+import "./chunk-7ZZMD7UK.js";
 import {
   extractUsage,
   extractUsageFromResponse,
@@ -401,18 +401,23 @@ import {
   assertRunAgentProfileCell,
   buildAgentInterfaceProfileCell,
   buildAgentProfileCell,
+  canonicalize,
+  evaluateHypothesis,
   groupRunsByAgentProfileCell,
+  hashJson,
   isRunRecord,
   modelHasSnapshot,
   parseRunRecordSafe,
   requireAgentProfileCell,
   roundTripRunRecord,
   runTaskScore,
+  signManifest,
   toAgentProfileJson,
   validateAgentProfileCell,
   validateRunRecord,
-  verifyAgentProfileCell
-} from "./chunk-NL33OXEP.js";
+  verifyAgentProfileCell,
+  verifyManifest
+} from "./chunk-2JX3CFMB.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -422,13 +427,6 @@ import {
   isSandboxSpan,
   isToolSpan
 } from "./chunk-MA6HLL3S.js";
-import {
-  canonicalize,
-  evaluateHypothesis,
-  hashJson,
-  signManifest,
-  verifyManifest
-} from "./chunk-VSMTAMNK.js";
 import {
   AgentEvalError,
   CaptureIntegrityError,
@@ -10648,17 +10646,28 @@ function rawPassOf(record) {
 }
 function passOf(record, score, threshold) {
   const rawPass = rawPassOf(record);
-  if (rawPass !== null) return rawPass && !record.failureMode;
-  return score >= threshold && !record.failureMode;
+  const hasTaskFailure = record.failureClass !== void 0 && record.failureClass !== "success";
+  if (rawPass !== null) return rawPass && !hasTaskFailure;
+  return score >= threshold && !hasTaskFailure;
 }
-function failureModeOf(record, score, threshold) {
-  if (record.failureMode) return record.failureMode;
+function failureOf(record, score, threshold) {
+  if (record.failureClass !== void 0 && record.failureClass !== "success") {
+    return {
+      failureClass: record.failureClass,
+      failureDetail: record.failureMode ?? null
+    };
+  }
   const belowThreshold = `quality-below-threshold: ${Math.round(score * 100)}% < ${Math.round(threshold * 100)}%`;
   if (rawPassOf(record) === false) {
-    return score < threshold ? belowThreshold : "product-pass-failed";
+    return {
+      failureClass: "unknown",
+      failureDetail: score < threshold ? belowThreshold : "product-pass-failed"
+    };
+  }
+  if (passOf(record, score, threshold)) {
+    return { failureClass: null, failureDetail: null };
   }
-  if (passOf(record, score, threshold)) return null;
-  return belowThreshold;
+  return { failureClass: "unknown", failureDetail: belowThreshold };
 }
 function numericDimensions(record) {
   const dimensions = {};
@@ -10775,6 +10784,7 @@ function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifac
   const opts = resolveOptions(options);
   const runtime = runtimeResolution(record, opts);
   const score = scoreOf(record);
+  const failure = failureOf(record, score, opts.passThreshold);
   const armId = armIdOf(record);
   const inputTokens = record.tokenUsage.input;
   const outputTokens = record.tokenUsage.output;
@@ -10809,7 +10819,7 @@ function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifac
       pass: passOf(record, score, opts.passThreshold),
       score,
       dimensions,
-      failureMode: failureModeOf(record, score, opts.passThreshold)
+      ...failure
     },
     usage: {
       inputTokens,
@@ -11024,6 +11034,13 @@ function expectString(value, path) {
     fail(path, "must be a non-empty string");
   return value;
 }
+function expectFailureClass(value, path) {
+  const parsed = expectString(value, path);
+  if (parsed === "success" || !FAILURE_CLASSES.includes(parsed)) {
+    fail(path, `expected a non-success FailureClass, received ${JSON.stringify(parsed)}`);
+  }
+  return parsed;
+}
 function expectBoolean(value, path) {
   if (typeof value !== "boolean") fail(path, "must be a boolean");
   return value;
@@ -11166,6 +11183,18 @@ function validateProductBenchmarkRecord(value) {
   const usage = expectObject(obj.usage, "record.usage");
   const integrity = expectObject(obj.integrity, "record.integrity");
   const artifacts = expectObject(obj.artifacts, "record.artifacts");
+  const outcomePass = expectBoolean(outcome.pass, "record.outcome.pass");
+  const failureClass = outcome.failureClass === null ? null : expectFailureClass(outcome.failureClass, "record.outcome.failureClass");
+  const failureDetail = outcome.failureDetail === null ? null : expectString(outcome.failureDetail, "record.outcome.failureDetail");
+  if (outcomePass && (failureClass !== null || failureDetail !== null)) {
+    fail("record.outcome", "a passing row cannot carry failure evidence");
+  }
+  if (!outcomePass && failureClass === null) {
+    fail("record.outcome.failureClass", "a failed row requires a failure class");
+  }
+  if (failureClass === null && failureDetail !== null) {
+    fail("record.outcome.failureDetail", "failure detail requires a failure class");
+  }
   const record = {
     schemaVersion: 1,
     projectId: expectString(obj.projectId, "record.projectId"),
@@ -11191,10 +11220,11 @@ function validateProductBenchmarkRecord(value) {
       version: expectString(backend.version, "record.backend.version")
     },
     outcome: {
-      pass: expectBoolean(outcome.pass, "record.outcome.pass"),
+      pass: outcomePass,
       score: expectNumber(outcome.score, "record.outcome.score", { min: 0, max: 1 }),
       dimensions: expectDimensions(outcome.dimensions, "record.outcome.dimensions"),
-      failureMode: outcome.failureMode === null ? null : expectString(outcome.failureMode, "record.outcome.failureMode")
+      failureClass,
+      failureDetail
     },
     usage: {
       inputTokens: expectNumber(usage.inputTokens, "record.usage.inputTokens", {