@tangle-network/agent-eval 0.126.7 → 0.128.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +6 -2
  3. package/dist/analyst/index.d.ts +52 -43
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/belief-state/index.d.ts +35 -40
  6. package/dist/belief-state/index.js +2 -2
  7. package/dist/benchmarks/index.d.ts +30 -12
  8. package/dist/benchmarks/index.js +12 -11
  9. package/dist/builder-eval/index.js +1 -1
  10. package/dist/campaign/index.d.ts +441 -160
  11. package/dist/campaign/index.js +11 -10
  12. package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
  13. package/dist/chunk-2JX3CFMB.js.map +1 -0
  14. package/dist/chunk-2MKQIFS4.js +183 -0
  15. package/dist/chunk-2MKQIFS4.js.map +1 -0
  16. package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
  17. package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
  18. package/dist/chunk-7ZZMD7UK.js.map +1 -0
  19. package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
  20. package/dist/chunk-BYT7ELPS.js.map +1 -0
  21. package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
  22. package/dist/chunk-DPUHNQLN.js +232 -0
  23. package/dist/chunk-DPUHNQLN.js.map +1 -0
  24. package/dist/chunk-DRYIUNWY.js +622 -0
  25. package/dist/chunk-DRYIUNWY.js.map +1 -0
  26. package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
  27. package/dist/chunk-EJGRPCO3.js.map +1 -0
  28. package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
  29. package/dist/chunk-EOSZT7PL.js.map +1 -0
  30. package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
  31. package/dist/chunk-EZJEIH2R.js.map +1 -0
  32. package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
  33. package/dist/chunk-IHQDPH7D.js.map +1 -0
  34. package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
  35. package/dist/chunk-MHELPNRP.js.map +1 -0
  36. package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
  37. package/dist/chunk-NACAGYSY.js.map +1 -0
  38. package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
  39. package/dist/chunk-NKAGIDE2.js.map +1 -0
  40. package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
  41. package/dist/chunk-NYLOYM6N.js.map +1 -0
  42. package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
  43. package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
  44. package/dist/chunk-P6FYH6K4.js.map +1 -0
  45. package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
  46. package/dist/chunk-PBE2LOSS.js.map +1 -0
  47. package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
  48. package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
  49. package/dist/chunk-SZLVEKMJ.js.map +1 -0
  50. package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
  51. package/dist/chunk-TBL77AUT.js.map +1 -0
  52. package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
  53. package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
  54. package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
  55. package/dist/chunk-UB2LOJ6Q.js.map +1 -0
  56. package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
  57. package/dist/chunk-UWZZKKU7.js.map +1 -0
  58. package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
  59. package/dist/chunk-VLOATJQ2.js.map +1 -0
  60. package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
  61. package/dist/chunk-XDWDC2MP.js.map +1 -0
  62. package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
  63. package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
  64. package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
  65. package/dist/cli.js +4 -4
  66. package/dist/contract/index.d.ts +438 -138
  67. package/dist/contract/index.js +259 -113
  68. package/dist/contract/index.js.map +1 -1
  69. package/dist/control.d.ts +56 -39
  70. package/dist/control.js +2 -3
  71. package/dist/hosted/index.d.ts +152 -51
  72. package/dist/hosted/index.js +24 -1
  73. package/dist/index.d.ts +399 -277
  74. package/dist/index.js +363 -244
  75. package/dist/index.js.map +1 -1
  76. package/dist/meta-eval/index.d.ts +33 -36
  77. package/dist/meta-eval/index.js +2 -2
  78. package/dist/multishot/index.d.ts +1 -0
  79. package/dist/openapi.json +1 -1
  80. package/dist/pipelines/index.js +2 -2
  81. package/dist/reporting.d.ts +101 -80
  82. package/dist/reporting.js +7 -5
  83. package/dist/rl.d.ts +198 -142
  84. package/dist/rl.js +235 -127
  85. package/dist/rl.js.map +1 -1
  86. package/dist/rollout/index.d.ts +54 -67
  87. package/dist/rollout/index.js +6 -5
  88. package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
  89. package/dist/supervisor-run/index.d.ts +7 -15
  90. package/dist/supervisor-run/index.js +2 -2
  91. package/dist/traces.d.ts +72 -50
  92. package/dist/traces.js +9 -6
  93. package/dist/wire/index.d.ts +1 -0
  94. package/dist/wire/index.js +2 -2
  95. package/docs/concepts.md +18 -1
  96. package/docs/customer-journeys.md +7 -6
  97. package/docs/hosted-ingest-spec.md +37 -40
  98. package/docs/insight-report.md +78 -7
  99. package/docs/research-report-methodology.md +20 -21
  100. package/docs/rollout.md +13 -8
  101. package/package.json +4 -3
  102. package/dist/chunk-474LBSOX.js.map +0 -1
  103. package/dist/chunk-7VYEGHWF.js.map +0 -1
  104. package/dist/chunk-ARU2PZFM.js.map +0 -1
  105. package/dist/chunk-BTACVO2W.js.map +0 -1
  106. package/dist/chunk-DPZAEKA6.js.map +0 -1
  107. package/dist/chunk-E3IADPLG.js.map +0 -1
  108. package/dist/chunk-H5UD2323.js.map +0 -1
  109. package/dist/chunk-IILEIWGW.js.map +0 -1
  110. package/dist/chunk-IPYXE555.js.map +0 -1
  111. package/dist/chunk-IR3KBHOY.js.map +0 -1
  112. package/dist/chunk-J5SQWP6Y.js.map +0 -1
  113. package/dist/chunk-K6IAZZ6L.js.map +0 -1
  114. package/dist/chunk-M7AH34KV.js.map +0 -1
  115. package/dist/chunk-MAX3TN3C.js.map +0 -1
  116. package/dist/chunk-MOXWMGPC.js.map +0 -1
  117. package/dist/chunk-OCFJACJU.js.map +0 -1
  118. package/dist/chunk-P22LJ3Y2.js.map +0 -1
  119. package/dist/chunk-PJQFMIOX.js.map +0 -1
  120. package/dist/chunk-VSMTAMNK.js +0 -53
  121. package/dist/chunk-VSMTAMNK.js.map +0 -1
  122. package/dist/chunk-ZVCHKKOP.js.map +0 -1
  123. package/dist/chunk-ZZUXHH3R.js +0 -99
  124. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  125. /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
  126. /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
  127. /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
  128. /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
  129. /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
  130. /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
  131. /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
  132. /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
@@ -420,7 +420,8 @@ interface AgentProfileCell {
420
420
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
421
421
  * fields are exactly those the paper "Two Loops, Three Roles" requires
422
422
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
423
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
423
+ * holdout split tag. A task score is optional because execution-only records
424
+ * must preserve missing labels instead of converting errors into zero quality.
424
425
  *
425
426
  * This is intentionally NOT a replacement for the rich `Run` /
426
427
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -441,6 +442,13 @@ interface AgentProfileCell {
441
442
  /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
442
443
  * combined train+test pool that the optimizer is allowed to read. */
443
444
  type RunSplitTag = 'search' | 'dev' | 'holdout';
445
+ /**
446
+ * Explicit execution-lifecycle result for a run.
447
+ *
448
+ * This is separate from task quality (`outcome`) and failure classification.
449
+ * Producers set it only from root-run or process evidence.
450
+ */
451
+ type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
444
452
  interface RunTokenUsage {
445
453
  input: number;
446
454
  /** All generated tokens charged as output, including reasoning tokens. */
@@ -454,11 +462,6 @@ interface RunTokenUsage {
454
462
  }
455
463
  /**
456
464
  * How a run's USD amount was obtained.
457
- *
458
- * `costUsd` remains mandatory for wire compatibility. New producers should
459
- * always populate this discriminated union so a missing bill is never
460
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
461
- * the legacy `0` sentinel while this field carries the truthful null.
462
465
  */
463
466
  type RunCostProvenance = {
464
467
  kind: 'observed';
@@ -504,8 +507,8 @@ interface JudgeScoresRecord {
504
507
  perJudge: Record<string, Record<string, number>>;
505
508
  /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
506
509
  perDimMean: Record<string, number>;
507
- /** Composite mean across all dims and judges. Mirrors the score
508
- * the gate sees on `outcome.searchScore` / `holdoutScore`. */
510
+ /** Composite mean across successful judges. Mirrors the task score only
511
+ * when `failedJudges` is empty. */
509
512
  composite: number;
510
513
  /** Judges that errored or returned an unparseable verdict. Recorded
511
514
  * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -516,11 +519,11 @@ interface JudgeScoresRecord {
516
519
  notes?: string;
517
520
  }
518
521
  interface RunOutcome {
519
- /** Score on the search/optimization split. Optional because a
520
- * holdout-only evaluation only fills `holdoutScore`. */
522
+ /** Score on the search/optimization split. Optional for holdout-only and
523
+ * execution-only records. */
521
524
  searchScore?: number;
522
- /** Score on the held-out split. Optional because a search-only run
523
- * only fills `searchScore`. At least one must be present. */
525
+ /** Score on the held-out split. Optional for search-only and execution-only
526
+ * records. When both scores are absent, the run is explicitly unlabeled. */
524
527
  holdoutScore?: number;
525
528
  /** Bag of any other metric the run produced — judge dimensions,
526
529
  * pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -584,42 +587,36 @@ interface RunRecord {
584
587
  wallMs: number;
585
588
  /** Time spent queued before execution started, if known. */
586
589
  queueMs?: number;
587
- /** Total USD cost. Mandatory runs without a cost number are
588
- * unbounded by definition and must not be admitted into the gate.
589
- * `0` is retained as the compatibility sentinel for an uncaptured amount;
590
- * inspect `costProvenance` before treating it as observed. */
591
- costUsd: number;
592
- /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
593
- * Optional only so existing serialized RunRecords remain valid. */
594
- costProvenance?: RunCostProvenance;
590
+ /** Total USD cost, or null when the producer could not capture one. */
591
+ costUsd: number | null;
592
+ /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
593
+ costProvenance: RunCostProvenance;
595
594
  /** Token usage breakdown. */
596
595
  tokenUsage: RunTokenUsage;
596
+ /** Root-run or process terminal result. Never inferred from a child span. */
597
+ terminalOutcome: RunTerminalOutcome;
598
+ /** Root-run or process failure reason. Valid only for a failed, cancelled,
599
+ * or incomplete terminal result; never populated from a child span. */
600
+ terminalFailureReason?: string;
597
601
  /** Judge-side metadata, if a judge was used. */
598
602
  judgeMetadata?: RunJudgeMetadata;
599
603
  /** Per-split scores + raw bag. */
600
604
  outcome: RunOutcome;
601
- /** Canonical, cross-agent failure class drawn from the shared
602
- * `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
603
- * "which failure dominates across the whole fleet" answerable in ONE
604
- * vocabulary — every agent classifies against the same enum. Producers
605
- * set it via the substrate classifier; leave unset only when the failure
606
- * genuinely can't be classified. */
605
+ /** Canonical task-failure class drawn from the shared
606
+ * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
607
+ * evidence. Execution errors belong in
608
+ * `outcome.raw.execution_error_count`. */
607
609
  failureClass?: FailureClass;
608
- /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
609
- * (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
610
- * The within-agent drill-down; `failureClass` is the cross-agent key. */
610
+ /** Free-form task-failure detail scoped under a non-success
611
+ * `failureClass`. It is invalid without that class. */
611
612
  failureMode?: string;
612
613
  /** Which split this run was drawn from. */
613
614
  splitTag: RunSplitTag;
614
615
  /**
615
- * Stable scenario identifier the run was scored against. Optional for
616
- * backwards compatibility, but **strongly recommended**: every primitive
617
- * that pairs runs by scenario (preferences, paired stats, BT tournament)
618
- * keys on this. The campaign artifact populates it canonically; legacy
619
- * runs without it fall back to inference from `outcome.raw.scenario_id`
620
- * or `experimentId`.
616
+ * Stable scenario identifier the run observed or was scored against.
617
+ * Comparison primitives match this identity rather than input order.
621
618
  */
622
- scenarioId?: string;
619
+ scenarioId: string;
623
620
  /**
624
621
  * Canonical identity for the agent profile cell that produced this row:
625
622
  * profile artifact hash plus optional harness/model/prompt/reporting
@@ -11,11 +11,11 @@ import {
11
11
  } from "../chunk-3RF76KTD.js";
12
12
  import {
13
13
  rubricPredictiveValidity
14
- } from "../chunk-X4UCIOTZ.js";
14
+ } from "../chunk-S5YLIBFX.js";
15
15
  import {
16
16
  pearsonR,
17
17
  spearmanR
18
- } from "../chunk-PJQFMIOX.js";
18
+ } from "../chunk-MHELPNRP.js";
19
19
  import {
20
20
  aggregateLlm,
21
21
  llmSpans
@@ -176,6 +176,7 @@ declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToo
176
176
  * OpenAI-compatible `/v1/chat/completions` client with:
177
177
  * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
178
178
  * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
179
+ * - One retry at temperature 1 when a model explicitly requires it.
179
180
  * - Graceful json_schema → json_object degrade on 400 with schema-reject body.
180
181
  * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
181
182
  * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
package/dist/openapi.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.1.0",
3
3
  "info": {
4
4
  "title": "@tangle-network/agent-eval — wire protocol",
5
- "version": "0.126.7",
5
+ "version": "0.128.0",
6
6
  "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
7
7
  "contact": {
8
8
  "name": "Tangle Network",
@@ -3,7 +3,7 @@ import {
3
3
  classifyFailure,
4
4
  compareToBaseline,
5
5
  computeToolUseMetrics
6
- } from "../chunk-QOTFXW5L.js";
6
+ } from "../chunk-P5W7RQKK.js";
7
7
  import {
8
8
  buildTrajectory
9
9
  } from "../chunk-RZTMDUO7.js";
@@ -13,7 +13,7 @@ import {
13
13
  import {
14
14
  interRaterReliability,
15
15
  pearsonR
16
- } from "../chunk-PJQFMIOX.js";
16
+ } from "../chunk-MHELPNRP.js";
17
17
  import {
18
18
  aggregateLlm,
19
19
  argHash,
@@ -31,7 +31,8 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
31
31
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
32
32
  * fields are exactly those the paper "Two Loops, Three Roles" requires
33
33
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
34
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
34
+ * holdout split tag. A task score is optional because execution-only records
35
+ * must preserve missing labels instead of converting errors into zero quality.
35
36
  *
36
37
  * This is intentionally NOT a replacement for the rich `Run` /
37
38
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -52,6 +53,13 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
52
53
  /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
53
54
  * combined train+test pool that the optimizer is allowed to read. */
54
55
  type RunSplitTag = 'search' | 'dev' | 'holdout';
56
+ /**
57
+ * Explicit execution-lifecycle result for a run.
58
+ *
59
+ * This is separate from task quality (`outcome`) and failure classification.
60
+ * Producers set it only from root-run or process evidence.
61
+ */
62
+ type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
55
63
  interface RunTokenUsage {
56
64
  input: number;
57
65
  /** All generated tokens charged as output, including reasoning tokens. */
@@ -65,11 +73,6 @@ interface RunTokenUsage {
65
73
  }
66
74
  /**
67
75
  * How a run's USD amount was obtained.
68
- *
69
- * `costUsd` remains mandatory for wire compatibility. New producers should
70
- * always populate this discriminated union so a missing bill is never
71
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
72
- * the legacy `0` sentinel while this field carries the truthful null.
73
76
  */
74
77
  type RunCostProvenance = {
75
78
  kind: 'observed';
@@ -115,8 +118,8 @@ interface JudgeScoresRecord {
115
118
  perJudge: Record<string, Record<string, number>>;
116
119
  /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
117
120
  perDimMean: Record<string, number>;
118
- /** Composite mean across all dims and judges. Mirrors the score
119
- * the gate sees on `outcome.searchScore` / `holdoutScore`. */
121
+ /** Composite mean across successful judges. Mirrors the task score only
122
+ * when `failedJudges` is empty. */
120
123
  composite: number;
121
124
  /** Judges that errored or returned an unparseable verdict. Recorded
122
125
  * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -127,11 +130,11 @@ interface JudgeScoresRecord {
127
130
  notes?: string;
128
131
  }
129
132
  interface RunOutcome {
130
- /** Score on the search/optimization split. Optional because a
131
- * holdout-only evaluation only fills `holdoutScore`. */
133
+ /** Score on the search/optimization split. Optional for holdout-only and
134
+ * execution-only records. */
132
135
  searchScore?: number;
133
- /** Score on the held-out split. Optional because a search-only run
134
- * only fills `searchScore`. At least one must be present. */
136
+ /** Score on the held-out split. Optional for search-only and execution-only
137
+ * records. When both scores are absent, the run is explicitly unlabeled. */
135
138
  holdoutScore?: number;
136
139
  /** Bag of any other metric the run produced — judge dimensions,
137
140
  * pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -195,42 +198,36 @@ interface RunRecord {
195
198
  wallMs: number;
196
199
  /** Time spent queued before execution started, if known. */
197
200
  queueMs?: number;
198
- /** Total USD cost. Mandatory runs without a cost number are
199
- * unbounded by definition and must not be admitted into the gate.
200
- * `0` is retained as the compatibility sentinel for an uncaptured amount;
201
- * inspect `costProvenance` before treating it as observed. */
202
- costUsd: number;
203
- /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
204
- * Optional only so existing serialized RunRecords remain valid. */
205
- costProvenance?: RunCostProvenance;
201
+ /** Total USD cost, or null when the producer could not capture one. */
202
+ costUsd: number | null;
203
+ /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
204
+ costProvenance: RunCostProvenance;
206
205
  /** Token usage breakdown. */
207
206
  tokenUsage: RunTokenUsage;
207
+ /** Root-run or process terminal result. Never inferred from a child span. */
208
+ terminalOutcome: RunTerminalOutcome;
209
+ /** Root-run or process failure reason. Valid only for a failed, cancelled,
210
+ * or incomplete terminal result; never populated from a child span. */
211
+ terminalFailureReason?: string;
208
212
  /** Judge-side metadata, if a judge was used. */
209
213
  judgeMetadata?: RunJudgeMetadata;
210
214
  /** Per-split scores + raw bag. */
211
215
  outcome: RunOutcome;
212
- /** Canonical, cross-agent failure class drawn from the shared
213
- * `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
214
- * "which failure dominates across the whole fleet" answerable in ONE
215
- * vocabulary — every agent classifies against the same enum. Producers
216
- * set it via the substrate classifier; leave unset only when the failure
217
- * genuinely can't be classified. */
216
+ /** Canonical task-failure class drawn from the shared
217
+ * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
218
+ * evidence. Execution errors belong in
219
+ * `outcome.raw.execution_error_count`. */
218
220
  failureClass?: FailureClass;
219
- /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
220
- * (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
221
- * The within-agent drill-down; `failureClass` is the cross-agent key. */
221
+ /** Free-form task-failure detail scoped under a non-success
222
+ * `failureClass`. It is invalid without that class. */
222
223
  failureMode?: string;
223
224
  /** Which split this run was drawn from. */
224
225
  splitTag: RunSplitTag;
225
226
  /**
226
- * Stable scenario identifier the run was scored against. Optional for
227
- * backwards compatibility, but **strongly recommended**: every primitive
228
- * that pairs runs by scenario (preferences, paired stats, BT tournament)
229
- * keys on this. The campaign artifact populates it canonically; legacy
230
- * runs without it fall back to inference from `outcome.raw.scenario_id`
231
- * or `experimentId`.
227
+ * Stable scenario identifier the run observed or was scored against.
228
+ * Comparison primitives match this identity rather than input order.
232
229
  */
233
- scenarioId?: string;
230
+ scenarioId: string;
234
231
  /**
235
232
  * Canonical identity for the agent profile cell that produced this row:
236
233
  * profile artifact hash plus optional harness/model/prompt/reporting
@@ -568,35 +565,39 @@ interface DatasetManifest {
568
565
  * specific promotion path (still useful for replay-style evals).
569
566
  */
570
567
 
571
- type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
568
+ type HeldOutGateRejectionCode = 'few_runs' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
572
569
  interface GateEvidence {
573
570
  /** Number of paired (candidate, baseline) holdout observations used. */
574
571
  productiveRuns: number;
575
- /** Median of (candidate baseline) paired holdout deltas. */
576
- medianPairedDelta: number;
577
- /** Bootstrap CI on the median paired holdout delta. */
572
+ /** Candidate holdout rows with no baseline row at the same work identity. */
573
+ unpairedCandidateRuns: number;
574
+ /** Baseline holdout rows with no candidate row at the same work identity. */
575
+ unpairedBaselineRuns: number;
576
+ /** Median of paired holdout deltas, or null when there are no pairs. */
577
+ medianPairedDelta: number | null;
578
+ /** Bootstrap CI on the median paired holdout delta, if computed. */
578
579
  pairedCI: {
579
580
  low: number;
580
581
  high: number;
581
- };
582
- /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
583
- pairedPValue: number;
584
- /** Mean candidate score on the search split (NaN if none). */
585
- searchScore: number;
586
- /** Mean candidate score on the holdout split (NaN if none). */
587
- holdoutScore: number;
588
- /** Candidate (search − holdout) gap. */
589
- overfitGap: number;
590
- /** Baseline (search − holdout) gap. */
591
- baselineOverfitGap: number;
582
+ } | null;
583
+ /** Wilcoxon signed-rank p-value, if computed. */
584
+ pairedPValue: number | null;
585
+ /** Mean candidate score on the search split, or null when absent. */
586
+ searchScore: number | null;
587
+ /** Mean candidate score on the holdout split, or null when absent. */
588
+ holdoutScore: number | null;
589
+ /** Candidate (search − holdout) gap, or null when either side is absent. */
590
+ overfitGap: number | null;
591
+ /** Baseline (search − holdout) gap, or null when either side is absent. */
592
+ baselineOverfitGap: number | null;
592
593
  /** Median per-task USD cost across the candidate's runs. Recorded
593
594
  * even when no `costPerTaskCeiling` is configured so downstream
594
595
  * dashboards (intelligence.tangle.tools) can render \$/task per
595
596
  * generation regardless of gating policy. */
596
- medianCandidateCost: number;
597
+ medianCandidateCost: number | null;
597
598
  /** Median per-task USD cost across the baseline runs, for
598
599
  * symmetric reporting. */
599
- medianBaselineCost: number;
600
+ medianBaselineCost: number | null;
600
601
  }
601
602
  interface GateDecision {
602
603
  /** Final promote/no-promote verdict. */
@@ -647,7 +648,7 @@ interface ActionableSideInfo {
647
648
  metadata?: Record<string, unknown>;
648
649
  }
649
650
  type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
650
- type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
651
+ type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'reliability' | 'generalization' | 'diagnostics' | 'efficiency';
651
652
  interface ReleaseTraceEvidence {
652
653
  scenarioId: string;
653
654
  candidateId?: string;
@@ -657,7 +658,8 @@ interface ReleaseTraceEvidence {
657
658
  turnCount?: number;
658
659
  costUsd?: number;
659
660
  durationMs?: number;
660
- failureMode?: string;
661
+ /** Canonical task-failure class. Free-form detail belongs in ASI. */
662
+ failureClass?: FailureClass;
661
663
  asi?: ActionableSideInfo[];
662
664
  metadata?: Record<string, unknown>;
663
665
  }
@@ -694,7 +696,7 @@ interface ReleaseConfidenceInput {
694
696
  interface ReleaseConfidenceAxis {
695
697
  name: ReleaseConfidenceAxisName;
696
698
  status: ReleaseConfidenceStatus;
697
- score: number;
699
+ score: number | null;
698
700
  detail: string;
699
701
  }
700
702
  interface ReleaseConfidenceIssue {
@@ -705,22 +707,32 @@ interface ReleaseConfidenceIssue {
705
707
  }
706
708
  interface ReleaseConfidenceMetrics {
707
709
  scenarioCount: number;
710
+ /** Search rows with a finite search score. */
708
711
  searchRuns: number;
712
+ /** Holdout rows with a finite holdout score. */
709
713
  holdoutRuns: number;
710
- passRate: number;
711
- meanScore: number;
712
- searchMeanScore: number;
713
- holdoutMeanScore: number;
714
- overfitGap: number;
715
- meanCostUsd: number;
716
- p95WallMs: number;
714
+ /** Runs with neither a split-matched score nor an explicit task failure. */
715
+ unscoredRuns: number;
716
+ /** Run rows, or trace rows when no runs exist, with no classified terminal result. */
717
+ unclassifiedTerminalRuns: number;
718
+ /** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */
719
+ terminalFailureRuns: number;
720
+ /** Success fraction when every run or fallback trace row has a classified result. */
721
+ reliabilityRate: number | null;
722
+ passRate: number | null;
723
+ meanScore: number | null;
724
+ searchMeanScore: number | null;
725
+ holdoutMeanScore: number | null;
726
+ overfitGap: number | null;
727
+ meanCostUsd: number | null;
728
+ p95WallMs: number | null;
717
729
  failedRows: number;
718
730
  failuresWithAsi: number;
719
731
  singleShotTraces: number;
720
732
  multiShotTraces: number;
721
733
  splitCounts: Record<DatasetSplit, number>;
722
734
  domainCounts: Record<string, number>;
723
- failureModeCounts: Record<string, number>;
735
+ failureClassCounts: Partial<Record<FailureClass, number>>;
724
736
  responsibleSurfaceCounts: Record<string, number>;
725
737
  }
726
738
  interface ReleaseConfidenceScorecard {
@@ -994,7 +1006,7 @@ interface FailureClusterReport {
994
1006
  */
995
1007
 
996
1008
  interface SummaryTableOptions {
997
- /** Comparator candidate id. Wilcoxon + Cohen's d are computed
1009
+ /** Comparator candidate id. Wilcoxon + paired Cohen's dz are computed
998
1010
  * versus this candidate. Required for paired stats columns. */
999
1011
  comparator?: string;
1000
1012
  /** Which split to read scores from. Default 'holdout'. */
@@ -1010,10 +1022,16 @@ interface SummaryTableRow {
1010
1022
  mean: number;
1011
1023
  ciLow: number;
1012
1024
  ciHigh: number;
1013
- /** BH-adjusted q-value vs comparator. NaN if no comparator. */
1014
- qValue: number;
1015
- /** Cohen's d vs comparator. NaN if no comparator. */
1016
- cohensD: number;
1025
+ /** BH-adjusted q-value vs comparator, or null when unavailable. */
1026
+ qValue: number | null;
1027
+ /** Paired Cohen's dz vs comparator, or null when the paired variance is zero. */
1028
+ cohensD: number | null;
1029
+ /** Matched observations used for paired comparison, or null on the comparator row. */
1030
+ pairedN: number | null;
1031
+ /** Candidate observations without a comparator match. */
1032
+ unpairedCandidateN: number | null;
1033
+ /** Comparator observations without a candidate match. */
1034
+ unpairedComparatorN: number | null;
1017
1035
  }
1018
1036
  interface SummaryTable {
1019
1037
  rows: SummaryTableRow[];
@@ -1025,7 +1043,7 @@ interface SummaryTable {
1025
1043
  /**
1026
1044
  * Table 1 helper. Buckets runs by `candidateId`, computes mean +
1027
1045
  * bootstrap CI on the chosen split, and (when a comparator is given)
1028
- * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
1046
+ * BH-adjusted Wilcoxon p + paired Cohen's dz versus that comparator.
1029
1047
  */
1030
1048
  declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
1031
1049
  interface ParetoPoint {
@@ -1041,7 +1059,7 @@ interface ParetoPoint {
1041
1059
  onFrontier: boolean;
1042
1060
  /** Optional gate verdict for this candidate, if a `GateDecision`
1043
1061
  * for it was passed in. */
1044
- gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
1062
+ gate?: 'promote' | 'reject';
1045
1063
  }
1046
1064
  interface ParetoFigureSpec {
1047
1065
  kind: 'pareto-cost-quality';
@@ -1076,12 +1094,16 @@ interface GainDistributionFigureSpec {
1076
1094
  split: 'search' | 'holdout';
1077
1095
  /** Number of pairs used. */
1078
1096
  n: number;
1097
+ /** Candidate rows without a comparator match. */
1098
+ unpairedCandidateN: number;
1099
+ /** Comparator rows without a candidate match. */
1100
+ unpairedComparatorN: number;
1079
1101
  bins: GainDistributionBin[];
1080
- median: number;
1102
+ median: number | null;
1081
1103
  ci: {
1082
1104
  low: number;
1083
1105
  high: number;
1084
- };
1106
+ } | null;
1085
1107
  }
1086
1108
  interface GainDistributionOptions {
1087
1109
  /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
@@ -1178,8 +1200,8 @@ interface ResearchReportCandidate {
1178
1200
  mean: number;
1179
1201
  ciLow: number;
1180
1202
  ciHigh: number;
1181
- qValue: number;
1182
- cohensD: number;
1203
+ qValue: number | null;
1204
+ cohensD: number | null;
1183
1205
  meanDeltaVsComparator: number | null;
1184
1206
  pairedN: number;
1185
1207
  medianGain: number | null;
@@ -1189,10 +1211,9 @@ interface ResearchReportCandidate {
1189
1211
  high: number;
1190
1212
  } | null;
1191
1213
  /**
1192
- * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
1193
- * from the same resamples that produce the gain CI; interpretable as
1194
- * "fraction of resamples in which the candidate beats the comparator on
1195
- * matched pairs."
1214
+ * Bayesian-bootstrap posterior summaries on the paired mean delta.
1215
+ * Dirichlet(1, ..., 1) weights represent uncertainty over the empirical
1216
+ * distribution of matched deltas.
1196
1217
  */
1197
1218
  prGreaterThanZero: number | null;
1198
1219
  prInRope: number | null;
@@ -1260,7 +1281,7 @@ interface ResearchReport {
1260
1281
  * - `summaryTable` marginal stats with BH-FDR-adjusted q-values
1261
1282
  * - `paretoChart` cost-vs-quality frontier with gate overlay
1262
1283
  * - `gainHistogram` per-candidate paired-delta distribution
1263
- * - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
1284
+ * - paired posterior (this file): bootstrap CI on median, Bayesian-bootstrap Pr(Δ>0),
1264
1285
  * Pr(Δ∈ROPE), MDE at the configured power
1265
1286
  *
1266
1287
  * Decisions are made on paired evidence — never on marginal means alone —
package/dist/reporting.js CHANGED
@@ -4,10 +4,10 @@ import {
4
4
  evaluateReleaseConfidence,
5
5
  judgeReplayGate,
6
6
  renderReleaseReport
7
- } from "./chunk-MOXWMGPC.js";
7
+ } from "./chunk-XDWDC2MP.js";
8
8
  import {
9
9
  rubricPredictiveValidity
10
- } from "./chunk-X4UCIOTZ.js";
10
+ } from "./chunk-S5YLIBFX.js";
11
11
  import {
12
12
  evaluateInterimReleaseConfidence,
13
13
  pairedEvalueSequence
@@ -18,13 +18,15 @@ import {
18
18
  paretoChart,
19
19
  researchReport,
20
20
  summaryTable
21
- } from "./chunk-DPZAEKA6.js";
21
+ } from "./chunk-VLOATJQ2.js";
22
+ import "./chunk-DPUHNQLN.js";
22
23
  import {
23
24
  benjaminiHochberg,
24
25
  pairedBootstrap,
25
26
  wilcoxonSignedRank
26
- } from "./chunk-PJQFMIOX.js";
27
- import "./chunk-VSMTAMNK.js";
27
+ } from "./chunk-MHELPNRP.js";
28
+ import "./chunk-2JX3CFMB.js";
29
+ import "./chunk-MA6HLL3S.js";
28
30
  import "./chunk-ONWEPEDO.js";
29
31
  import "./chunk-PZ5AY32C.js";
30
32
  export {