@tangle-network/agent-eval 0.127.0 → 0.128.1

package/dist/meta-eval/index.d.ts

@@ -603,15 +603,12 @@ interface RunRecord {
     /** Per-split scores + raw bag. */
     outcome: RunOutcome;
     /** Canonical task-failure class drawn from the shared
-     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
-     *  "which failure dominates across the whole fleet" answerable in ONE
-     *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it only from task-result evidence. Execution errors belong in
-     *  `outcome.raw.execution_error_count`, even when the run later fails. */
+     *  `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
+     *  evidence. Execution errors belong in
+     *  `outcome.raw.execution_error_count`. */
     failureClass?: FailureClass;
-    /** Free-form task-failure detail, scoped UNDER `failureClass`
-     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  Do not populate this from a child execution error alone. */
+    /** Free-form task-failure detail scoped under a non-success
+     *  `failureClass`. It is invalid without that class. */
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.127.0",
+    "version": "0.128.1",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/reporting.d.ts

@@ -214,15 +214,12 @@ interface RunRecord {
     /** Per-split scores + raw bag. */
     outcome: RunOutcome;
     /** Canonical task-failure class drawn from the shared
-     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
-     *  "which failure dominates across the whole fleet" answerable in ONE
-     *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it only from task-result evidence. Execution errors belong in
-     *  `outcome.raw.execution_error_count`, even when the run later fails. */
+     *  `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
+     *  evidence. Execution errors belong in
+     *  `outcome.raw.execution_error_count`. */
     failureClass?: FailureClass;
-    /** Free-form task-failure detail, scoped UNDER `failureClass`
-     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  Do not populate this from a child execution error alone. */
+    /** Free-form task-failure detail scoped under a non-success
+     *  `failureClass`. It is invalid without that class. */
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;
@@ -661,7 +658,8 @@ interface ReleaseTraceEvidence {
     turnCount?: number;
     costUsd?: number;
     durationMs?: number;
-    failureMode?: string;
+    /** Canonical task-failure class. Free-form detail belongs in ASI. */
+    failureClass?: FailureClass;
     asi?: ActionableSideInfo[];
     metadata?: Record<string, unknown>;
 }
@@ -734,7 +732,7 @@ interface ReleaseConfidenceMetrics {
     multiShotTraces: number;
     splitCounts: Record<DatasetSplit, number>;
     domainCounts: Record<string, number>;
-    failureModeCounts: Record<string, number>;
+    failureClassCounts: Partial<Record<FailureClass, number>>;
     responsibleSurfaceCounts: Record<string, number>;
 }
 interface ReleaseConfidenceScorecard {

package/dist/reporting.js

@@ -4,7 +4,7 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-Y5CYOL42.js";
+} from "./chunk-XDWDC2MP.js";
 import {
   rubricPredictiveValidity
 } from "./chunk-S5YLIBFX.js";
@@ -18,14 +18,15 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-IMGJZBFG.js";
+} from "./chunk-VLOATJQ2.js";
 import "./chunk-DPUHNQLN.js";
 import {
   benjaminiHochberg,
   pairedBootstrap,
   wilcoxonSignedRank
 } from "./chunk-MHELPNRP.js";
-import "./chunk-VSMTAMNK.js";
+import "./chunk-2JX3CFMB.js";
+import "./chunk-MA6HLL3S.js";
 import "./chunk-ONWEPEDO.js";
 import "./chunk-PZ5AY32C.js";
 export {

package/dist/rl.d.ts

@@ -416,15 +416,12 @@ interface RunRecord {
     /** Per-split scores + raw bag. */
     outcome: RunOutcome;
     /** Canonical task-failure class drawn from the shared
-     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
-     *  "which failure dominates across the whole fleet" answerable in ONE
-     *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it only from task-result evidence. Execution errors belong in
-     *  `outcome.raw.execution_error_count`, even when the run later fails. */
+     *  `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
+     *  evidence. Execution errors belong in
+     *  `outcome.raw.execution_error_count`. */
     failureClass?: FailureClass;
-    /** Free-form task-failure detail, scoped UNDER `failureClass`
-     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  Do not populate this from a child execution error alone. */
+    /** Free-form task-failure detail scoped under a non-success
+     *  `failureClass`. It is invalid without that class. */
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;
@@ -442,6 +439,23 @@ interface RunRecord {
      */
     agentProfile?: AgentProfileCell;
 }
+/**
+ * Canonical task-result classification.
+ *
+ * A producer may omit classification, record explicit success, or attach
+ * domain-specific detail to a non-success class. Detail can never stand alone.
+ * Execution errors belong in `outcome.raw.execution_error_count`.
+ */
+type RunTaskFailure = {
+    failureClass?: undefined;
+    failureMode?: undefined;
+} | {
+    failureClass: 'success';
+    failureMode?: undefined;
+} | {
+    failureClass: Exclude<FailureClass, 'success'>;
+    failureMode?: string;
+};
 
 /**
  * Adaptive curriculum / active scenario selection.
@@ -3231,7 +3245,7 @@ interface CampaignRunContext<V> {
      */
     llmOpts: LlmClientOptions;
 }
-interface CampaignRunOutcome {
+interface CampaignRunOutcomeFields {
     /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
     pass: boolean;
     /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
@@ -3249,12 +3263,6 @@ interface CampaignRunOutcome {
     configHash: string;
     /** Optional extra numeric metrics to land in `outcome.raw`. */
     raw?: Record<string, number>;
-    /** Canonical cross-agent failure class from the shared `FAILURE_CLASSES`
-     *  taxonomy. Propagated to `RunRecord.failureClass` so campaign runs
-     *  aggregate failures in the same vocabulary as every other producer. */
-    failureClass?: FailureClass;
-    /** Optional free-form failure detail, scoped under `failureClass`. */
-    failureMode?: string;
     /** Optional judge metadata when a judge was used. */
     judgeMetadata?: RunJudgeMetadata;
     /**
@@ -3270,6 +3278,8 @@ interface CampaignRunOutcome {
      */
     agentProfile?: AgentProfileCell | AgentProfileCellInput;
 }
+/** Campaign result with the same task-failure invariant as `RunRecord`. */
+type CampaignRunOutcome = CampaignRunOutcomeFields & RunTaskFailure;
 type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
 type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
 interface EvalCampaignOptions<V> {

package/dist/rl.js

@@ -10,16 +10,16 @@ import {
 } from "./chunk-3RF76KTD.js";
 import {
   runEvalCampaign
-} from "./chunk-CLELFRQG.js";
+} from "./chunk-TBL77AUT.js";
 import {
   detectRewardHacking,
   extractVerifiableReward,
   extractVerifiableRewardsFromRecords,
   filterDeterministicallyRewarded
-} from "./chunk-424XHVKV.js";
+} from "./chunk-NYLOYM6N.js";
 import {
   campaignCellToRunRecord
-} from "./chunk-ZPUEEM5U.js";
+} from "./chunk-2MKQIFS4.js";
 import "./chunk-PBE2LOSS.js";
 import {
   rubricPredictiveValidity
@@ -27,7 +27,7 @@ import {
 import {
   evaluateInterimReleaseConfidence
 } from "./chunk-MAZ26DC7.js";
-import "./chunk-IMGJZBFG.js";
+import "./chunk-VLOATJQ2.js";
 import "./chunk-DPUHNQLN.js";
 import {
   benjaminiHochberg,
@@ -45,9 +45,8 @@ import "./chunk-PC4UYEBM.js";
 import "./chunk-VQMK5FMP.js";
 import {
   runTaskScore
-} from "./chunk-NL33OXEP.js";
+} from "./chunk-2JX3CFMB.js";
 import "./chunk-MA6HLL3S.js";
-import "./chunk-VSMTAMNK.js";
 import {
   ValidationError
 } from "./chunk-ONWEPEDO.js";
@@ -1412,7 +1411,10 @@ function verificationReportToRunRecord(report, ctx, opts = {}) {
     tokenUsage: { input: 0, output: 0 },
     terminalOutcome: "succeeded",
     outcome,
-    failureMode: firstScoredFailure ? `layer_${firstScoredFailure.layer}_fail` : void 0,
+    ...firstScoredFailure ? {
+      failureClass: "unknown",
+      failureMode: `layer_${firstScoredFailure.layer}_fail`
+    } : {},
     splitTag,
     scenarioId: ctx.scenarioId
   };