@tangle-network/agent-eval 0.126.6 → 0.127.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/README.md +2 -2
  3. package/dist/analyst/index.d.ts +52 -40
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/belief-state/index.d.ts +34 -36
  6. package/dist/belief-state/index.js +2 -2
  7. package/dist/benchmarks/index.d.ts +30 -12
  8. package/dist/benchmarks/index.js +12 -10
  9. package/dist/builder-eval/index.js +1 -1
  10. package/dist/campaign/index.d.ts +432 -146
  11. package/dist/campaign/index.js +11 -9
  12. package/dist/{chunk-ARU2PZFM.js → chunk-424XHVKV.js} +43 -23
  13. package/dist/chunk-424XHVKV.js.map +1 -0
  14. package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
  15. package/dist/{chunk-BTACVO2W.js → chunk-65QETEKG.js} +542 -258
  16. package/dist/chunk-65QETEKG.js.map +1 -0
  17. package/dist/{chunk-CGG5SLH3.js → chunk-ARCFIY7U.js} +2 -2
  18. package/dist/{chunk-P22LJ3Y2.js → chunk-CLELFRQG.js} +6 -4
  19. package/dist/chunk-CLELFRQG.js.map +1 -0
  20. package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
  21. package/dist/chunk-DPUHNQLN.js +232 -0
  22. package/dist/chunk-DPUHNQLN.js.map +1 -0
  23. package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
  24. package/dist/chunk-EJGRPCO3.js.map +1 -0
  25. package/dist/{chunk-K6IAZZ6L.js → chunk-FA3MFYNA.js} +132 -110
  26. package/dist/chunk-FA3MFYNA.js.map +1 -0
  27. package/dist/chunk-IG35YDN3.js +614 -0
  28. package/dist/chunk-IG35YDN3.js.map +1 -0
  29. package/dist/{chunk-DPZAEKA6.js → chunk-IMGJZBFG.js} +84 -56
  30. package/dist/chunk-IMGJZBFG.js.map +1 -0
  31. package/dist/{chunk-H5UD2323.js → chunk-IN455XKS.js} +53 -1
  32. package/dist/chunk-IN455XKS.js.map +1 -0
  33. package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
  34. package/dist/chunk-MHELPNRP.js.map +1 -0
  35. package/dist/{chunk-IILEIWGW.js → chunk-NL33OXEP.js} +41 -36
  36. package/dist/chunk-NL33OXEP.js.map +1 -0
  37. package/dist/{chunk-474LBSOX.js → chunk-OD5NQ2CS.js} +41 -16
  38. package/dist/chunk-OD5NQ2CS.js.map +1 -0
  39. package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
  40. package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
  41. package/dist/chunk-P6FYH6K4.js.map +1 -0
  42. package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
  43. package/dist/chunk-PBE2LOSS.js.map +1 -0
  44. package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js} +50 -224
  45. package/dist/chunk-PRB53E3J.js.map +1 -0
  46. package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
  47. package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
  48. package/dist/chunk-SZLVEKMJ.js.map +1 -0
  49. package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
  50. package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
  51. package/dist/{chunk-M7AH34KV.js → chunk-UTA2TGC4.js} +32 -16
  52. package/dist/chunk-UTA2TGC4.js.map +1 -0
  53. package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
  54. package/dist/chunk-UWZZKKU7.js.map +1 -0
  55. package/dist/{chunk-ZVCHKKOP.js → chunk-WED5KLZQ.js} +39 -15
  56. package/dist/chunk-WED5KLZQ.js.map +1 -0
  57. package/dist/{chunk-OCFJACJU.js → chunk-X6VOPMLD.js} +74 -14
  58. package/dist/chunk-X6VOPMLD.js.map +1 -0
  59. package/dist/{chunk-MOXWMGPC.js → chunk-Y5CYOL42.js} +159 -70
  60. package/dist/chunk-Y5CYOL42.js.map +1 -0
  61. package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
  62. package/dist/chunk-ZPUEEM5U.js +179 -0
  63. package/dist/chunk-ZPUEEM5U.js.map +1 -0
  64. package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
  65. package/dist/cli.js +4 -4
  66. package/dist/contract/index.d.ts +464 -126
  67. package/dist/contract/index.js +425 -211
  68. package/dist/contract/index.js.map +1 -1
  69. package/dist/control.d.ts +34 -32
  70. package/dist/control.js +2 -2
  71. package/dist/hosted/index.d.ts +142 -41
  72. package/dist/hosted/index.js +23 -1
  73. package/dist/index.d.ts +365 -256
  74. package/dist/index.js +315 -226
  75. package/dist/index.js.map +1 -1
  76. package/dist/meta-eval/index.d.ts +32 -32
  77. package/dist/meta-eval/index.js +2 -2
  78. package/dist/multishot/index.d.ts +1 -0
  79. package/dist/openapi.json +1 -1
  80. package/dist/pipelines/index.js +2 -2
  81. package/dist/reporting.d.ts +97 -74
  82. package/dist/reporting.js +5 -4
  83. package/dist/rl.d.ts +177 -131
  84. package/dist/rl.js +232 -126
  85. package/dist/rl.js.map +1 -1
  86. package/dist/rollout/index.d.ts +53 -63
  87. package/dist/rollout/index.js +7 -5
  88. package/dist/{run-campaign-FFRM3RH5.js → run-campaign-3A2EBYBE.js} +4 -3
  89. package/dist/supervisor-run/index.d.ts +7 -15
  90. package/dist/supervisor-run/index.js +2 -2
  91. package/dist/traces.d.ts +66 -41
  92. package/dist/traces.js +9 -5
  93. package/dist/wire/index.d.ts +1 -0
  94. package/dist/wire/index.js +2 -2
  95. package/docs/concepts.md +18 -1
  96. package/docs/hosted-ingest-spec.md +37 -40
  97. package/docs/insight-report.md +57 -7
  98. package/docs/research-report-methodology.md +20 -21
  99. package/docs/rollout.md +13 -8
  100. package/package.json +21 -14
  101. package/dist/chunk-474LBSOX.js.map +0 -1
  102. package/dist/chunk-7VYEGHWF.js.map +0 -1
  103. package/dist/chunk-ARU2PZFM.js.map +0 -1
  104. package/dist/chunk-BTACVO2W.js.map +0 -1
  105. package/dist/chunk-DPZAEKA6.js.map +0 -1
  106. package/dist/chunk-E3IADPLG.js.map +0 -1
  107. package/dist/chunk-H5UD2323.js.map +0 -1
  108. package/dist/chunk-IILEIWGW.js.map +0 -1
  109. package/dist/chunk-IPYXE555.js.map +0 -1
  110. package/dist/chunk-IR3KBHOY.js.map +0 -1
  111. package/dist/chunk-J5SQWP6Y.js.map +0 -1
  112. package/dist/chunk-K6IAZZ6L.js.map +0 -1
  113. package/dist/chunk-M7AH34KV.js.map +0 -1
  114. package/dist/chunk-MAX3TN3C.js.map +0 -1
  115. package/dist/chunk-MOXWMGPC.js.map +0 -1
  116. package/dist/chunk-OCFJACJU.js.map +0 -1
  117. package/dist/chunk-P22LJ3Y2.js.map +0 -1
  118. package/dist/chunk-PJQFMIOX.js.map +0 -1
  119. package/dist/chunk-ZVCHKKOP.js.map +0 -1
  120. package/dist/chunk-ZZUXHH3R.js +0 -99
  121. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  122. /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
  123. /package/dist/{chunk-CGG5SLH3.js.map → chunk-ARCFIY7U.js.map} +0 -0
  124. /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
  125. /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
  126. /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
  127. /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
  128. /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
  129. /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-3A2EBYBE.js.map} +0 -0
@@ -420,7 +420,8 @@ interface AgentProfileCell {
420
420
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
421
421
  * fields are exactly those the paper "Two Loops, Three Roles" requires
422
422
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
423
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
423
+ * holdout split tag. A task score is optional because execution-only records
424
+ * must preserve missing labels instead of converting errors into zero quality.
424
425
  *
425
426
  * This is intentionally NOT a replacement for the rich `Run` /
426
427
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -441,6 +442,13 @@ interface AgentProfileCell {
441
442
  /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
442
443
  * combined train+test pool that the optimizer is allowed to read. */
443
444
  type RunSplitTag = 'search' | 'dev' | 'holdout';
445
+ /**
446
+ * Explicit execution-lifecycle result for a run.
447
+ *
448
+ * This is separate from task quality (`outcome`) and failure classification.
449
+ * Producers set it only from root-run or process evidence.
450
+ */
451
+ type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
444
452
  interface RunTokenUsage {
445
453
  input: number;
446
454
  /** All generated tokens charged as output, including reasoning tokens. */
@@ -454,11 +462,6 @@ interface RunTokenUsage {
454
462
  }
455
463
  /**
456
464
  * How a run's USD amount was obtained.
457
- *
458
- * `costUsd` remains mandatory for wire compatibility. New producers should
459
- * always populate this discriminated union so a missing bill is never
460
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
461
- * the legacy `0` sentinel while this field carries the truthful null.
462
465
  */
463
466
  type RunCostProvenance = {
464
467
  kind: 'observed';
@@ -504,8 +507,8 @@ interface JudgeScoresRecord {
504
507
  perJudge: Record<string, Record<string, number>>;
505
508
  /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
506
509
  perDimMean: Record<string, number>;
507
- /** Composite mean across all dims and judges. Mirrors the score
508
- * the gate sees on `outcome.searchScore` / `holdoutScore`. */
510
+ /** Composite mean across successful judges. Mirrors the task score only
511
+ * when `failedJudges` is empty. */
509
512
  composite: number;
510
513
  /** Judges that errored or returned an unparseable verdict. Recorded
511
514
  * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -516,11 +519,11 @@ interface JudgeScoresRecord {
516
519
  notes?: string;
517
520
  }
518
521
  interface RunOutcome {
519
- /** Score on the search/optimization split. Optional because a
520
- * holdout-only evaluation only fills `holdoutScore`. */
522
+ /** Score on the search/optimization split. Optional for holdout-only and
523
+ * execution-only records. */
521
524
  searchScore?: number;
522
- /** Score on the held-out split. Optional because a search-only run
523
- * only fills `searchScore`. At least one must be present. */
525
+ /** Score on the held-out split. Optional for search-only and execution-only
526
+ * records. When both scores are absent, the run is explicitly unlabeled. */
524
527
  holdoutScore?: number;
525
528
  /** Bag of any other metric the run produced — judge dimensions,
526
529
  * pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -584,42 +587,39 @@ interface RunRecord {
584
587
  wallMs: number;
585
588
  /** Time spent queued before execution started, if known. */
586
589
  queueMs?: number;
587
- /** Total USD cost. Mandatory runs without a cost number are
588
- * unbounded by definition and must not be admitted into the gate.
589
- * `0` is retained as the compatibility sentinel for an uncaptured amount;
590
- * inspect `costProvenance` before treating it as observed. */
591
- costUsd: number;
592
- /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
593
- * Optional only so existing serialized RunRecords remain valid. */
594
- costProvenance?: RunCostProvenance;
590
+ /** Total USD cost, or null when the producer could not capture one. */
591
+ costUsd: number | null;
592
+ /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
593
+ costProvenance: RunCostProvenance;
595
594
  /** Token usage breakdown. */
596
595
  tokenUsage: RunTokenUsage;
596
+ /** Root-run or process terminal result. Never inferred from a child span. */
597
+ terminalOutcome: RunTerminalOutcome;
598
+ /** Root-run or process failure reason. Valid only for a failed, cancelled,
599
+ * or incomplete terminal result; never populated from a child span. */
600
+ terminalFailureReason?: string;
597
601
  /** Judge-side metadata, if a judge was used. */
598
602
  judgeMetadata?: RunJudgeMetadata;
599
603
  /** Per-split scores + raw bag. */
600
604
  outcome: RunOutcome;
601
- /** Canonical, cross-agent failure class drawn from the shared
605
+ /** Canonical task-failure class drawn from the shared
602
606
  * `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
603
607
  * "which failure dominates across the whole fleet" answerable in ONE
604
608
  * vocabulary — every agent classifies against the same enum. Producers
605
- * set it via the substrate classifier; leave unset only when the failure
606
- * genuinely can't be classified. */
609
+ * set it only from task-result evidence. Execution errors belong in
610
+ * `outcome.raw.execution_error_count`, even when the run later fails. */
607
611
  failureClass?: FailureClass;
608
- /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
612
+ /** Free-form task-failure detail, scoped UNDER `failureClass`
609
613
  * (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
610
- * The within-agent drill-down; `failureClass` is the cross-agent key. */
614
+ * Do not populate this from a child execution error alone. */
611
615
  failureMode?: string;
612
616
  /** Which split this run was drawn from. */
613
617
  splitTag: RunSplitTag;
614
618
  /**
615
- * Stable scenario identifier the run was scored against. Optional for
616
- * backwards compatibility, but **strongly recommended**: every primitive
617
- * that pairs runs by scenario (preferences, paired stats, BT tournament)
618
- * keys on this. The campaign artifact populates it canonically; legacy
619
- * runs without it fall back to inference from `outcome.raw.scenario_id`
620
- * or `experimentId`.
619
+ * Stable scenario identifier the run observed or was scored against.
620
+ * Comparison primitives match this identity rather than input order.
621
621
  */
622
- scenarioId?: string;
622
+ scenarioId: string;
623
623
  /**
624
624
  * Canonical identity for the agent profile cell that produced this row:
625
625
  * profile artifact hash plus optional harness/model/prompt/reporting
@@ -11,11 +11,11 @@ import {
11
11
  } from "../chunk-3RF76KTD.js";
12
12
  import {
13
13
  rubricPredictiveValidity
14
- } from "../chunk-X4UCIOTZ.js";
14
+ } from "../chunk-S5YLIBFX.js";
15
15
  import {
16
16
  pearsonR,
17
17
  spearmanR
18
- } from "../chunk-PJQFMIOX.js";
18
+ } from "../chunk-MHELPNRP.js";
19
19
  import {
20
20
  aggregateLlm,
21
21
  llmSpans
@@ -176,6 +176,7 @@ declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToo
176
176
  * OpenAI-compatible `/v1/chat/completions` client with:
177
177
  * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
178
178
  * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
179
+ * - One retry at temperature 1 when a model explicitly requires it.
179
180
  * - Graceful json_schema → json_object degrade on 400 with schema-reject body.
180
181
  * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
181
182
  * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
package/dist/openapi.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.1.0",
3
3
  "info": {
4
4
  "title": "@tangle-network/agent-eval — wire protocol",
5
- "version": "0.126.6",
5
+ "version": "0.127.0",
6
6
  "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
7
7
  "contact": {
8
8
  "name": "Tangle Network",
@@ -3,7 +3,7 @@ import {
3
3
  classifyFailure,
4
4
  compareToBaseline,
5
5
  computeToolUseMetrics
6
- } from "../chunk-QOTFXW5L.js";
6
+ } from "../chunk-P5W7RQKK.js";
7
7
  import {
8
8
  buildTrajectory
9
9
  } from "../chunk-RZTMDUO7.js";
@@ -13,7 +13,7 @@ import {
13
13
  import {
14
14
  interRaterReliability,
15
15
  pearsonR
16
- } from "../chunk-PJQFMIOX.js";
16
+ } from "../chunk-MHELPNRP.js";
17
17
  import {
18
18
  aggregateLlm,
19
19
  argHash,
@@ -31,7 +31,8 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
31
31
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
32
32
  * fields are exactly those the paper "Two Loops, Three Roles" requires
33
33
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
34
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
34
+ * holdout split tag. A task score is optional because execution-only records
35
+ * must preserve missing labels instead of converting errors into zero quality.
35
36
  *
36
37
  * This is intentionally NOT a replacement for the rich `Run` /
37
38
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -52,6 +53,13 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
52
53
  /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
53
54
  * combined train+test pool that the optimizer is allowed to read. */
54
55
  type RunSplitTag = 'search' | 'dev' | 'holdout';
56
+ /**
57
+ * Explicit execution-lifecycle result for a run.
58
+ *
59
+ * This is separate from task quality (`outcome`) and failure classification.
60
+ * Producers set it only from root-run or process evidence.
61
+ */
62
+ type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
55
63
  interface RunTokenUsage {
56
64
  input: number;
57
65
  /** All generated tokens charged as output, including reasoning tokens. */
@@ -65,11 +73,6 @@ interface RunTokenUsage {
65
73
  }
66
74
  /**
67
75
  * How a run's USD amount was obtained.
68
- *
69
- * `costUsd` remains mandatory for wire compatibility. New producers should
70
- * always populate this discriminated union so a missing bill is never
71
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
72
- * the legacy `0` sentinel while this field carries the truthful null.
73
76
  */
74
77
  type RunCostProvenance = {
75
78
  kind: 'observed';
@@ -115,8 +118,8 @@ interface JudgeScoresRecord {
115
118
  perJudge: Record<string, Record<string, number>>;
116
119
  /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
117
120
  perDimMean: Record<string, number>;
118
- /** Composite mean across all dims and judges. Mirrors the score
119
- * the gate sees on `outcome.searchScore` / `holdoutScore`. */
121
+ /** Composite mean across successful judges. Mirrors the task score only
122
+ * when `failedJudges` is empty. */
120
123
  composite: number;
121
124
  /** Judges that errored or returned an unparseable verdict. Recorded
122
125
  * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -127,11 +130,11 @@ interface JudgeScoresRecord {
127
130
  notes?: string;
128
131
  }
129
132
  interface RunOutcome {
130
- /** Score on the search/optimization split. Optional because a
131
- * holdout-only evaluation only fills `holdoutScore`. */
133
+ /** Score on the search/optimization split. Optional for holdout-only and
134
+ * execution-only records. */
132
135
  searchScore?: number;
133
- /** Score on the held-out split. Optional because a search-only run
134
- * only fills `searchScore`. At least one must be present. */
136
+ /** Score on the held-out split. Optional for search-only and execution-only
137
+ * records. When both scores are absent, the run is explicitly unlabeled. */
135
138
  holdoutScore?: number;
136
139
  /** Bag of any other metric the run produced — judge dimensions,
137
140
  * pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -195,42 +198,39 @@ interface RunRecord {
195
198
  wallMs: number;
196
199
  /** Time spent queued before execution started, if known. */
197
200
  queueMs?: number;
198
- /** Total USD cost. Mandatory runs without a cost number are
199
- * unbounded by definition and must not be admitted into the gate.
200
- * `0` is retained as the compatibility sentinel for an uncaptured amount;
201
- * inspect `costProvenance` before treating it as observed. */
202
- costUsd: number;
203
- /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
204
- * Optional only so existing serialized RunRecords remain valid. */
205
- costProvenance?: RunCostProvenance;
201
+ /** Total USD cost, or null when the producer could not capture one. */
202
+ costUsd: number | null;
203
+ /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
204
+ costProvenance: RunCostProvenance;
206
205
  /** Token usage breakdown. */
207
206
  tokenUsage: RunTokenUsage;
207
+ /** Root-run or process terminal result. Never inferred from a child span. */
208
+ terminalOutcome: RunTerminalOutcome;
209
+ /** Root-run or process failure reason. Valid only for a failed, cancelled,
210
+ * or incomplete terminal result; never populated from a child span. */
211
+ terminalFailureReason?: string;
208
212
  /** Judge-side metadata, if a judge was used. */
209
213
  judgeMetadata?: RunJudgeMetadata;
210
214
  /** Per-split scores + raw bag. */
211
215
  outcome: RunOutcome;
212
- /** Canonical, cross-agent failure class drawn from the shared
216
+ /** Canonical task-failure class drawn from the shared
213
217
  * `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
214
218
  * "which failure dominates across the whole fleet" answerable in ONE
215
219
  * vocabulary — every agent classifies against the same enum. Producers
216
- * set it via the substrate classifier; leave unset only when the failure
217
- * genuinely can't be classified. */
220
+ * set it only from task-result evidence. Execution errors belong in
221
+ * `outcome.raw.execution_error_count`, even when the run later fails. */
218
222
  failureClass?: FailureClass;
219
- /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
223
+ /** Free-form task-failure detail, scoped UNDER `failureClass`
220
224
  * (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
221
- * The within-agent drill-down; `failureClass` is the cross-agent key. */
225
+ * Do not populate this from a child execution error alone. */
222
226
  failureMode?: string;
223
227
  /** Which split this run was drawn from. */
224
228
  splitTag: RunSplitTag;
225
229
  /**
226
- * Stable scenario identifier the run was scored against. Optional for
227
- * backwards compatibility, but **strongly recommended**: every primitive
228
- * that pairs runs by scenario (preferences, paired stats, BT tournament)
229
- * keys on this. The campaign artifact populates it canonically; legacy
230
- * runs without it fall back to inference from `outcome.raw.scenario_id`
231
- * or `experimentId`.
230
+ * Stable scenario identifier the run observed or was scored against.
231
+ * Comparison primitives match this identity rather than input order.
232
232
  */
233
- scenarioId?: string;
233
+ scenarioId: string;
234
234
  /**
235
235
  * Canonical identity for the agent profile cell that produced this row:
236
236
  * profile artifact hash plus optional harness/model/prompt/reporting
@@ -568,35 +568,39 @@ interface DatasetManifest {
568
568
  * specific promotion path (still useful for replay-style evals).
569
569
  */
570
570
 
571
- type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
571
+ type HeldOutGateRejectionCode = 'few_runs' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
572
572
  interface GateEvidence {
573
573
  /** Number of paired (candidate, baseline) holdout observations used. */
574
574
  productiveRuns: number;
575
- /** Median of (candidate baseline) paired holdout deltas. */
576
- medianPairedDelta: number;
577
- /** Bootstrap CI on the median paired holdout delta. */
575
+ /** Candidate holdout rows with no baseline row at the same work identity. */
576
+ unpairedCandidateRuns: number;
577
+ /** Baseline holdout rows with no candidate row at the same work identity. */
578
+ unpairedBaselineRuns: number;
579
+ /** Median of paired holdout deltas, or null when there are no pairs. */
580
+ medianPairedDelta: number | null;
581
+ /** Bootstrap CI on the median paired holdout delta, if computed. */
578
582
  pairedCI: {
579
583
  low: number;
580
584
  high: number;
581
- };
582
- /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
583
- pairedPValue: number;
584
- /** Mean candidate score on the search split (NaN if none). */
585
- searchScore: number;
586
- /** Mean candidate score on the holdout split (NaN if none). */
587
- holdoutScore: number;
588
- /** Candidate (search − holdout) gap. */
589
- overfitGap: number;
590
- /** Baseline (search − holdout) gap. */
591
- baselineOverfitGap: number;
585
+ } | null;
586
+ /** Wilcoxon signed-rank p-value, if computed. */
587
+ pairedPValue: number | null;
588
+ /** Mean candidate score on the search split, or null when absent. */
589
+ searchScore: number | null;
590
+ /** Mean candidate score on the holdout split, or null when absent. */
591
+ holdoutScore: number | null;
592
+ /** Candidate (search − holdout) gap, or null when either side is absent. */
593
+ overfitGap: number | null;
594
+ /** Baseline (search − holdout) gap, or null when either side is absent. */
595
+ baselineOverfitGap: number | null;
592
596
  /** Median per-task USD cost across the candidate's runs. Recorded
593
597
  * even when no `costPerTaskCeiling` is configured so downstream
594
598
  * dashboards (intelligence.tangle.tools) can render \$/task per
595
599
  * generation regardless of gating policy. */
596
- medianCandidateCost: number;
600
+ medianCandidateCost: number | null;
597
601
  /** Median per-task USD cost across the baseline runs, for
598
602
  * symmetric reporting. */
599
- medianBaselineCost: number;
603
+ medianBaselineCost: number | null;
600
604
  }
601
605
  interface GateDecision {
602
606
  /** Final promote/no-promote verdict. */
@@ -647,7 +651,7 @@ interface ActionableSideInfo {
647
651
  metadata?: Record<string, unknown>;
648
652
  }
649
653
  type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
650
- type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
654
+ type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'reliability' | 'generalization' | 'diagnostics' | 'efficiency';
651
655
  interface ReleaseTraceEvidence {
652
656
  scenarioId: string;
653
657
  candidateId?: string;
@@ -694,7 +698,7 @@ interface ReleaseConfidenceInput {
694
698
  interface ReleaseConfidenceAxis {
695
699
  name: ReleaseConfidenceAxisName;
696
700
  status: ReleaseConfidenceStatus;
697
- score: number;
701
+ score: number | null;
698
702
  detail: string;
699
703
  }
700
704
  interface ReleaseConfidenceIssue {
@@ -705,15 +709,25 @@ interface ReleaseConfidenceIssue {
705
709
  }
706
710
  interface ReleaseConfidenceMetrics {
707
711
  scenarioCount: number;
712
+ /** Search rows with a finite search score. */
708
713
  searchRuns: number;
714
+ /** Holdout rows with a finite holdout score. */
709
715
  holdoutRuns: number;
710
- passRate: number;
711
- meanScore: number;
712
- searchMeanScore: number;
713
- holdoutMeanScore: number;
714
- overfitGap: number;
715
- meanCostUsd: number;
716
- p95WallMs: number;
716
+ /** Runs with neither a split-matched score nor an explicit task failure. */
717
+ unscoredRuns: number;
718
+ /** Run rows, or trace rows when no runs exist, with no classified terminal result. */
719
+ unclassifiedTerminalRuns: number;
720
+ /** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */
721
+ terminalFailureRuns: number;
722
+ /** Success fraction when every run or fallback trace row has a classified result. */
723
+ reliabilityRate: number | null;
724
+ passRate: number | null;
725
+ meanScore: number | null;
726
+ searchMeanScore: number | null;
727
+ holdoutMeanScore: number | null;
728
+ overfitGap: number | null;
729
+ meanCostUsd: number | null;
730
+ p95WallMs: number | null;
717
731
  failedRows: number;
718
732
  failuresWithAsi: number;
719
733
  singleShotTraces: number;
@@ -994,7 +1008,7 @@ interface FailureClusterReport {
994
1008
  */
995
1009
 
996
1010
  interface SummaryTableOptions {
997
- /** Comparator candidate id. Wilcoxon + Cohen's d are computed
1011
+ /** Comparator candidate id. Wilcoxon + paired Cohen's dz are computed
998
1012
  * versus this candidate. Required for paired stats columns. */
999
1013
  comparator?: string;
1000
1014
  /** Which split to read scores from. Default 'holdout'. */
@@ -1010,10 +1024,16 @@ interface SummaryTableRow {
1010
1024
  mean: number;
1011
1025
  ciLow: number;
1012
1026
  ciHigh: number;
1013
- /** BH-adjusted q-value vs comparator. NaN if no comparator. */
1014
- qValue: number;
1015
- /** Cohen's d vs comparator. NaN if no comparator. */
1016
- cohensD: number;
1027
+ /** BH-adjusted q-value vs comparator, or null when unavailable. */
1028
+ qValue: number | null;
1029
+ /** Paired Cohen's dz vs comparator, or null when the paired variance is zero. */
1030
+ cohensD: number | null;
1031
+ /** Matched observations used for paired comparison, or null on the comparator row. */
1032
+ pairedN: number | null;
1033
+ /** Candidate observations without a comparator match. */
1034
+ unpairedCandidateN: number | null;
1035
+ /** Comparator observations without a candidate match. */
1036
+ unpairedComparatorN: number | null;
1017
1037
  }
1018
1038
  interface SummaryTable {
1019
1039
  rows: SummaryTableRow[];
@@ -1025,7 +1045,7 @@ interface SummaryTable {
1025
1045
  /**
1026
1046
  * Table 1 helper. Buckets runs by `candidateId`, computes mean +
1027
1047
  * bootstrap CI on the chosen split, and (when a comparator is given)
1028
- * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
1048
+ * BH-adjusted Wilcoxon p + paired Cohen's dz versus that comparator.
1029
1049
  */
1030
1050
  declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
1031
1051
  interface ParetoPoint {
@@ -1041,7 +1061,7 @@ interface ParetoPoint {
1041
1061
  onFrontier: boolean;
1042
1062
  /** Optional gate verdict for this candidate, if a `GateDecision`
1043
1063
  * for it was passed in. */
1044
- gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
1064
+ gate?: 'promote' | 'reject';
1045
1065
  }
1046
1066
  interface ParetoFigureSpec {
1047
1067
  kind: 'pareto-cost-quality';
@@ -1076,12 +1096,16 @@ interface GainDistributionFigureSpec {
1076
1096
  split: 'search' | 'holdout';
1077
1097
  /** Number of pairs used. */
1078
1098
  n: number;
1099
+ /** Candidate rows without a comparator match. */
1100
+ unpairedCandidateN: number;
1101
+ /** Comparator rows without a candidate match. */
1102
+ unpairedComparatorN: number;
1079
1103
  bins: GainDistributionBin[];
1080
- median: number;
1104
+ median: number | null;
1081
1105
  ci: {
1082
1106
  low: number;
1083
1107
  high: number;
1084
- };
1108
+ } | null;
1085
1109
  }
1086
1110
  interface GainDistributionOptions {
1087
1111
  /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
@@ -1178,8 +1202,8 @@ interface ResearchReportCandidate {
1178
1202
  mean: number;
1179
1203
  ciLow: number;
1180
1204
  ciHigh: number;
1181
- qValue: number;
1182
- cohensD: number;
1205
+ qValue: number | null;
1206
+ cohensD: number | null;
1183
1207
  meanDeltaVsComparator: number | null;
1184
1208
  pairedN: number;
1185
1209
  medianGain: number | null;
@@ -1189,10 +1213,9 @@ interface ResearchReportCandidate {
1189
1213
  high: number;
1190
1214
  } | null;
1191
1215
  /**
1192
- * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
1193
- * from the same resamples that produce the gain CI; interpretable as
1194
- * "fraction of resamples in which the candidate beats the comparator on
1195
- * matched pairs."
1216
+ * Bayesian-bootstrap posterior summaries on the paired mean delta.
1217
+ * Dirichlet(1, ..., 1) weights represent uncertainty over the empirical
1218
+ * distribution of matched deltas.
1196
1219
  */
1197
1220
  prGreaterThanZero: number | null;
1198
1221
  prInRope: number | null;
@@ -1260,7 +1283,7 @@ interface ResearchReport {
1260
1283
  * - `summaryTable` marginal stats with BH-FDR-adjusted q-values
1261
1284
  * - `paretoChart` cost-vs-quality frontier with gate overlay
1262
1285
  * - `gainHistogram` per-candidate paired-delta distribution
1263
- * - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
1286
+ * - paired posterior (this file): bootstrap CI on median, Bayesian-bootstrap Pr(Δ>0),
1264
1287
  * Pr(Δ∈ROPE), MDE at the configured power
1265
1288
  *
1266
1289
  * Decisions are made on paired evidence — never on marginal means alone —
package/dist/reporting.js CHANGED
@@ -4,10 +4,10 @@ import {
4
4
  evaluateReleaseConfidence,
5
5
  judgeReplayGate,
6
6
  renderReleaseReport
7
- } from "./chunk-MOXWMGPC.js";
7
+ } from "./chunk-Y5CYOL42.js";
8
8
  import {
9
9
  rubricPredictiveValidity
10
- } from "./chunk-X4UCIOTZ.js";
10
+ } from "./chunk-S5YLIBFX.js";
11
11
  import {
12
12
  evaluateInterimReleaseConfidence,
13
13
  pairedEvalueSequence
@@ -18,12 +18,13 @@ import {
18
18
  paretoChart,
19
19
  researchReport,
20
20
  summaryTable
21
- } from "./chunk-DPZAEKA6.js";
21
+ } from "./chunk-IMGJZBFG.js";
22
+ import "./chunk-DPUHNQLN.js";
22
23
  import {
23
24
  benjaminiHochberg,
24
25
  pairedBootstrap,
25
26
  wilcoxonSignedRank
26
- } from "./chunk-PJQFMIOX.js";
27
+ } from "./chunk-MHELPNRP.js";
27
28
  import "./chunk-VSMTAMNK.js";
28
29
  import "./chunk-ONWEPEDO.js";
29
30
  import "./chunk-PZ5AY32C.js";