@tangle-network/agent-eval 0.126.7 → 0.128.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +6 -2
  3. package/dist/analyst/index.d.ts +52 -43
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/belief-state/index.d.ts +35 -40
  6. package/dist/belief-state/index.js +2 -2
  7. package/dist/benchmarks/index.d.ts +30 -12
  8. package/dist/benchmarks/index.js +12 -11
  9. package/dist/builder-eval/index.js +1 -1
  10. package/dist/campaign/index.d.ts +441 -160
  11. package/dist/campaign/index.js +11 -10
  12. package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
  13. package/dist/chunk-2JX3CFMB.js.map +1 -0
  14. package/dist/chunk-2MKQIFS4.js +183 -0
  15. package/dist/chunk-2MKQIFS4.js.map +1 -0
  16. package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
  17. package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
  18. package/dist/chunk-7ZZMD7UK.js.map +1 -0
  19. package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
  20. package/dist/chunk-BYT7ELPS.js.map +1 -0
  21. package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
  22. package/dist/chunk-DPUHNQLN.js +232 -0
  23. package/dist/chunk-DPUHNQLN.js.map +1 -0
  24. package/dist/chunk-DRYIUNWY.js +622 -0
  25. package/dist/chunk-DRYIUNWY.js.map +1 -0
  26. package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
  27. package/dist/chunk-EJGRPCO3.js.map +1 -0
  28. package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
  29. package/dist/chunk-EOSZT7PL.js.map +1 -0
  30. package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
  31. package/dist/chunk-EZJEIH2R.js.map +1 -0
  32. package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
  33. package/dist/chunk-IHQDPH7D.js.map +1 -0
  34. package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
  35. package/dist/chunk-MHELPNRP.js.map +1 -0
  36. package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
  37. package/dist/chunk-NACAGYSY.js.map +1 -0
  38. package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
  39. package/dist/chunk-NKAGIDE2.js.map +1 -0
  40. package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
  41. package/dist/chunk-NYLOYM6N.js.map +1 -0
  42. package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
  43. package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
  44. package/dist/chunk-P6FYH6K4.js.map +1 -0
  45. package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
  46. package/dist/chunk-PBE2LOSS.js.map +1 -0
  47. package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
  48. package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
  49. package/dist/chunk-SZLVEKMJ.js.map +1 -0
  50. package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
  51. package/dist/chunk-TBL77AUT.js.map +1 -0
  52. package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
  53. package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
  54. package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
  55. package/dist/chunk-UB2LOJ6Q.js.map +1 -0
  56. package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
  57. package/dist/chunk-UWZZKKU7.js.map +1 -0
  58. package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
  59. package/dist/chunk-VLOATJQ2.js.map +1 -0
  60. package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
  61. package/dist/chunk-XDWDC2MP.js.map +1 -0
  62. package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
  63. package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
  64. package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
  65. package/dist/cli.js +4 -4
  66. package/dist/contract/index.d.ts +438 -138
  67. package/dist/contract/index.js +259 -113
  68. package/dist/contract/index.js.map +1 -1
  69. package/dist/control.d.ts +56 -39
  70. package/dist/control.js +2 -3
  71. package/dist/hosted/index.d.ts +152 -51
  72. package/dist/hosted/index.js +24 -1
  73. package/dist/index.d.ts +399 -277
  74. package/dist/index.js +363 -244
  75. package/dist/index.js.map +1 -1
  76. package/dist/meta-eval/index.d.ts +33 -36
  77. package/dist/meta-eval/index.js +2 -2
  78. package/dist/multishot/index.d.ts +1 -0
  79. package/dist/openapi.json +1 -1
  80. package/dist/pipelines/index.js +2 -2
  81. package/dist/reporting.d.ts +101 -80
  82. package/dist/reporting.js +7 -5
  83. package/dist/rl.d.ts +198 -142
  84. package/dist/rl.js +235 -127
  85. package/dist/rl.js.map +1 -1
  86. package/dist/rollout/index.d.ts +54 -67
  87. package/dist/rollout/index.js +6 -5
  88. package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
  89. package/dist/supervisor-run/index.d.ts +7 -15
  90. package/dist/supervisor-run/index.js +2 -2
  91. package/dist/traces.d.ts +72 -50
  92. package/dist/traces.js +9 -6
  93. package/dist/wire/index.d.ts +1 -0
  94. package/dist/wire/index.js +2 -2
  95. package/docs/concepts.md +18 -1
  96. package/docs/customer-journeys.md +7 -6
  97. package/docs/hosted-ingest-spec.md +37 -40
  98. package/docs/insight-report.md +78 -7
  99. package/docs/research-report-methodology.md +20 -21
  100. package/docs/rollout.md +13 -8
  101. package/package.json +4 -3
  102. package/dist/chunk-474LBSOX.js.map +0 -1
  103. package/dist/chunk-7VYEGHWF.js.map +0 -1
  104. package/dist/chunk-ARU2PZFM.js.map +0 -1
  105. package/dist/chunk-BTACVO2W.js.map +0 -1
  106. package/dist/chunk-DPZAEKA6.js.map +0 -1
  107. package/dist/chunk-E3IADPLG.js.map +0 -1
  108. package/dist/chunk-H5UD2323.js.map +0 -1
  109. package/dist/chunk-IILEIWGW.js.map +0 -1
  110. package/dist/chunk-IPYXE555.js.map +0 -1
  111. package/dist/chunk-IR3KBHOY.js.map +0 -1
  112. package/dist/chunk-J5SQWP6Y.js.map +0 -1
  113. package/dist/chunk-K6IAZZ6L.js.map +0 -1
  114. package/dist/chunk-M7AH34KV.js.map +0 -1
  115. package/dist/chunk-MAX3TN3C.js.map +0 -1
  116. package/dist/chunk-MOXWMGPC.js.map +0 -1
  117. package/dist/chunk-OCFJACJU.js.map +0 -1
  118. package/dist/chunk-P22LJ3Y2.js.map +0 -1
  119. package/dist/chunk-PJQFMIOX.js.map +0 -1
  120. package/dist/chunk-VSMTAMNK.js +0 -53
  121. package/dist/chunk-VSMTAMNK.js.map +0 -1
  122. package/dist/chunk-ZVCHKKOP.js.map +0 -1
  123. package/dist/chunk-ZZUXHH3R.js +0 -99
  124. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  125. /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
  126. /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
  127. /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
  128. /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
  129. /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
  130. /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
  131. /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
  132. /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
package/dist/control.d.ts CHANGED
@@ -411,6 +411,8 @@ type ControlDecision<TAction> = {
411
411
  reason: string;
412
412
  pass?: boolean;
413
413
  score?: number;
414
+ /** Canonical task-failure class when this stop represents a failed task. */
415
+ failureClass?: FailureClass;
414
416
  };
415
417
  interface StopDecision {
416
418
  stop: boolean;
@@ -770,7 +772,8 @@ interface AgentProfileCell {
770
772
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
771
773
  * fields are exactly those the paper "Two Loops, Three Roles" requires
772
774
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
773
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
775
+ * holdout split tag. A task score is optional because execution-only records
776
+ * must preserve missing labels instead of converting errors into zero quality.
774
777
  *
775
778
  * This is intentionally NOT a replacement for the rich `Run` /
776
779
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -791,6 +794,13 @@ interface AgentProfileCell {
791
794
  /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
792
795
  * combined train+test pool that the optimizer is allowed to read. */
793
796
  type RunSplitTag = 'search' | 'dev' | 'holdout';
797
+ /**
798
+ * Explicit execution-lifecycle result for a run.
799
+ *
800
+ * This is separate from task quality (`outcome`) and failure classification.
801
+ * Producers set it only from root-run or process evidence.
802
+ */
803
+ type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
794
804
  interface RunTokenUsage {
795
805
  input: number;
796
806
  /** All generated tokens charged as output, including reasoning tokens. */
@@ -804,11 +814,6 @@ interface RunTokenUsage {
804
814
  }
805
815
  /**
806
816
  * How a run's USD amount was obtained.
807
- *
808
- * `costUsd` remains mandatory for wire compatibility. New producers should
809
- * always populate this discriminated union so a missing bill is never
810
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
811
- * the legacy `0` sentinel while this field carries the truthful null.
812
817
  */
813
818
  type RunCostProvenance = {
814
819
  kind: 'observed';
@@ -854,8 +859,8 @@ interface JudgeScoresRecord {
854
859
  perJudge: Record<string, Record<string, number>>;
855
860
  /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
856
861
  perDimMean: Record<string, number>;
857
- /** Composite mean across all dims and judges. Mirrors the score
858
- * the gate sees on `outcome.searchScore` / `holdoutScore`. */
862
+ /** Composite mean across successful judges. Mirrors the task score only
863
+ * when `failedJudges` is empty. */
859
864
  composite: number;
860
865
  /** Judges that errored or returned an unparseable verdict. Recorded
861
866
  * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -866,11 +871,11 @@ interface JudgeScoresRecord {
866
871
  notes?: string;
867
872
  }
868
873
  interface RunOutcome {
869
- /** Score on the search/optimization split. Optional because a
870
- * holdout-only evaluation only fills `holdoutScore`. */
874
+ /** Score on the search/optimization split. Optional for holdout-only and
875
+ * execution-only records. */
871
876
  searchScore?: number;
872
- /** Score on the held-out split. Optional because a search-only run
873
- * only fills `searchScore`. At least one must be present. */
877
+ /** Score on the held-out split. Optional for search-only and execution-only
878
+ * records. When both scores are absent, the run is explicitly unlabeled. */
874
879
  holdoutScore?: number;
875
880
  /** Bag of any other metric the run produced — judge dimensions,
876
881
  * pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -934,42 +939,36 @@ interface RunRecord {
934
939
  wallMs: number;
935
940
  /** Time spent queued before execution started, if known. */
936
941
  queueMs?: number;
937
- /** Total USD cost. Mandatory runs without a cost number are
938
- * unbounded by definition and must not be admitted into the gate.
939
- * `0` is retained as the compatibility sentinel for an uncaptured amount;
940
- * inspect `costProvenance` before treating it as observed. */
941
- costUsd: number;
942
- /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
943
- * Optional only so existing serialized RunRecords remain valid. */
944
- costProvenance?: RunCostProvenance;
942
+ /** Total USD cost, or null when the producer could not capture one. */
943
+ costUsd: number | null;
944
+ /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
945
+ costProvenance: RunCostProvenance;
945
946
  /** Token usage breakdown. */
946
947
  tokenUsage: RunTokenUsage;
948
+ /** Root-run or process terminal result. Never inferred from a child span. */
949
+ terminalOutcome: RunTerminalOutcome;
950
+ /** Root-run or process failure reason. Valid only for a failed, cancelled,
951
+ * or incomplete terminal result; never populated from a child span. */
952
+ terminalFailureReason?: string;
947
953
  /** Judge-side metadata, if a judge was used. */
948
954
  judgeMetadata?: RunJudgeMetadata;
949
955
  /** Per-split scores + raw bag. */
950
956
  outcome: RunOutcome;
951
- /** Canonical, cross-agent failure class drawn from the shared
952
- * `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
953
- * "which failure dominates across the whole fleet" answerable in ONE
954
- * vocabulary — every agent classifies against the same enum. Producers
955
- * set it via the substrate classifier; leave unset only when the failure
956
- * genuinely can't be classified. */
957
+ /** Canonical task-failure class drawn from the shared
958
+ * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
959
+ * evidence. Execution errors belong in
960
+ * `outcome.raw.execution_error_count`. */
957
961
  failureClass?: FailureClass;
958
- /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
959
- * (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
960
- * The within-agent drill-down; `failureClass` is the cross-agent key. */
962
+ /** Free-form task-failure detail scoped under a non-success
963
+ * `failureClass`. It is invalid without that class. */
961
964
  failureMode?: string;
962
965
  /** Which split this run was drawn from. */
963
966
  splitTag: RunSplitTag;
964
967
  /**
965
- * Stable scenario identifier the run was scored against. Optional for
966
- * backwards compatibility, but **strongly recommended**: every primitive
967
- * that pairs runs by scenario (preferences, paired stats, BT tournament)
968
- * keys on this. The campaign artifact populates it canonically; legacy
969
- * runs without it fall back to inference from `outcome.raw.scenario_id`
970
- * or `experimentId`.
968
+ * Stable scenario identifier the run observed or was scored against.
969
+ * Comparison primitives match this identity rather than input order.
971
970
  */
972
- scenarioId?: string;
971
+ scenarioId: string;
973
972
  /**
974
973
  * Canonical identity for the agent profile cell that produced this row:
975
974
  * profile artifact hash plus optional harness/model/prompt/reporting
@@ -979,9 +978,27 @@ interface RunRecord {
979
978
  */
980
979
  agentProfile?: AgentProfileCell;
981
980
  }
981
+ /**
982
+ * Canonical task-result classification.
983
+ *
984
+ * A producer may omit classification, record explicit success, or attach
985
+ * domain-specific detail to a non-success class. Detail can never stand alone.
986
+ * Execution errors belong in `outcome.raw.execution_error_count`.
987
+ */
988
+ type RunTaskFailure = {
989
+ failureClass?: undefined;
990
+ failureMode?: undefined;
991
+ } | {
992
+ failureClass: 'success';
993
+ failureMode?: undefined;
994
+ } | {
995
+ failureClass: Exclude<FailureClass, 'success'>;
996
+ failureMode?: string;
997
+ };
982
998
 
983
999
  interface RunEvidenceMetadata {
984
1000
  experimentId: string;
1001
+ scenarioId: string;
985
1002
  candidateId: string;
986
1003
  seed: number;
987
1004
  model: string;
@@ -990,15 +1007,15 @@ interface RunEvidenceMetadata {
990
1007
  commitSha: string;
991
1008
  splitTag: RunSplitTag;
992
1009
  tokenUsage: RunTokenUsage;
1010
+ costProvenance: RunRecord['costProvenance'];
993
1011
  queueMs?: number;
994
1012
  judgeMetadata?: RunRecord['judgeMetadata'];
995
1013
  raw?: Record<string, number>;
996
1014
  }
997
- interface ControlRunToRunRecordOptions extends RunEvidenceMetadata {
1015
+ type ControlRunToRunRecordOptions = RunEvidenceMetadata & RunTaskFailure & {
998
1016
  runId?: string;
999
1017
  score?: number;
1000
- failureMode?: string;
1001
- }
1018
+ };
1002
1019
  /**
1003
1020
  * Project a completed control-loop run into the strict RunRecord shape used by
1004
1021
  * release gates, optimizer tables, and research reports.
package/dist/control.js CHANGED
@@ -10,11 +10,10 @@ import {
10
10
  stopOnNoProgress,
11
11
  stopOnRepeatedAction,
12
12
  subjectiveEval
13
- } from "./chunk-474LBSOX.js";
13
+ } from "./chunk-BYT7ELPS.js";
14
14
  import "./chunk-VQMK5FMP.js";
15
- import "./chunk-IILEIWGW.js";
15
+ import "./chunk-2JX3CFMB.js";
16
16
  import "./chunk-MA6HLL3S.js";
17
- import "./chunk-VSMTAMNK.js";
18
17
  import "./chunk-ONWEPEDO.js";
19
18
  import "./chunk-PZ5AY32C.js";
20
19
  export {
@@ -1,3 +1,41 @@
1
+ import { z } from 'zod';
2
+
3
+ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
4
+
5
+ /**
6
+ * Paper-grade RunRecord schema + runtime validator.
7
+ *
8
+ * Every run that participates in a promotion gate, paper table, or
9
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
10
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
11
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
12
+ * holdout split tag. A task score is optional because execution-only records
13
+ * must preserve missing labels instead of converting errors into zero quality.
14
+ *
15
+ * This is intentionally NOT a replacement for the rich `Run` /
16
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
17
+ * package. Those are runtime structures with full provenance. A
18
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
19
+ * row you'd put in a parquet file or paste into a notebook.
20
+ *
21
+ * Validate at the boundary:
22
+ *
23
+ * const rec = validateRunRecord(rawJson) // throws on missing
24
+ * const ok = isRunRecord(rawJson) // boolean check
25
+ * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }
26
+ *
27
+ * The validator runs in pure TS — zod is intentionally NOT a
28
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
29
+ */
30
+
31
+ /**
32
+ * Explicit execution-lifecycle result for a run.
33
+ *
34
+ * This is separate from task quality (`outcome`) and failure classification.
35
+ * Producers set it only from root-run or process evidence.
36
+ */
37
+ type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
38
+
1
39
  /**
2
40
  * Pass A substrate types — `runCampaign` is the one primitive every
3
41
  * eval flow composes from. Three contracts in this file:
@@ -102,7 +140,7 @@ interface ParetoPoint {
102
140
  onFrontier: boolean;
103
141
  /** Optional gate verdict for this candidate, if a `GateDecision`
104
142
  * for it was passed in. */
105
- gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
143
+ gate?: 'promote' | 'reject';
106
144
  }
107
145
  interface ParetoFigureSpec {
108
146
  kind: 'pareto-cost-quality';
@@ -176,8 +214,8 @@ interface InsightReport {
176
214
  /** Number of runs analyzed. */
177
215
  n: number;
178
216
  /** Runtime facts carried by the run records. These describe execution,
179
- * not task quality: duration, queueing, token categories, models, and
180
- * explicitly recorded failures. */
217
+ * not task quality: duration, queueing, token categories, models,
218
+ * execution errors, and terminal outcomes. */
181
219
  execution: ExecutionInsight;
182
220
  /** Composite-score distribution across all runs. Always present. */
183
221
  composite: ScalarDistribution;
@@ -235,11 +273,10 @@ interface InsightReport {
235
273
  * Surfaced metrics: composite, cost, duration, tokenUsage, plus any
236
274
  * per-dimension judge metric present in both windows. */
237
275
  priorPeriodComparison?: PriorPeriodComparison;
238
- /** Model-free failure-mode breakdown from `RunRecord.failureMode`, ranked
239
- * by count descending. Present when any run carries a `failureMode`.
240
- * Complements `failureClusters` (LLM-semantic) with the structured tags
241
- * the harness already recorded — actionable with no analyst wired. */
242
- failureModes?: FailureModeTally[];
276
+ /** Model-free task-failure breakdown from `RunRecord.failureClass`, ranked
277
+ * by count descending. Domain-specific `failureMode` detail is retained on
278
+ * each record but never creates a second aggregation vocabulary. */
279
+ failureClasses?: FailureClassTally[];
243
280
  /** Top-N actionable recommendations, ranked by priority. The packet's
244
281
  * human-readable layer; the numeric sections are the evidence. */
245
282
  recommendations: Recommendation[];
@@ -287,15 +324,46 @@ interface ExecutionInsight {
287
324
  events: number;
288
325
  reportingRuns: number;
289
326
  };
290
- /** Failure counts remain separate from outcome scores. `reportedErrorEvents`
291
- * sums `outcome.raw.error_span_count` only where a producer supplied it. */
292
- failures: {
327
+ /** Runs with explicit execution-error telemetry. This is independent of
328
+ * whether the root run ultimately succeeded, failed, or has no terminal
329
+ * evidence. */
330
+ executionErrors: {
293
331
  runs: number;
294
- fraction: number;
295
- reportedErrorEvents: number;
332
+ /** Share among runs that supplied an execution-error count.
333
+ * `null` when no run supplied error telemetry. */
334
+ fraction: number | null;
335
+ /** Execution-error events reported through the canonical count. */
336
+ events: number;
337
+ /** Runs that supplied an execution-error count, including explicit zeroes. */
296
338
  reportingRuns: number;
339
+ /** Exact sum of `outcome.raw.error_span_count`, kept separate from other errors. */
340
+ errorSpanEvents: number;
341
+ /** Runs that supplied `outcome.raw.error_span_count`, including explicit zeroes. */
342
+ errorSpanReportingRuns: number;
343
+ /**
344
+ * Error-telemetry coverage crossed with independently reported terminal
345
+ * outcomes. `unreported` is distinct from a reported zero.
346
+ */
347
+ byTerminalOutcome: Record<RunTerminalOutcome, ExecutionErrorOutcomeCell>;
348
+ };
349
+ /** Root-run or process outcomes. Missing `RunRecord.terminalOutcome` values
350
+ * count as `unknown`; child-span status never changes these counts. */
351
+ terminalOutcomes: {
352
+ succeeded: number;
353
+ failed: number;
354
+ cancelled: number;
355
+ incomplete: number;
356
+ unknown: number;
297
357
  };
298
358
  }
359
+ interface ExecutionErrorOutcomeCell {
360
+ /** Runs that explicitly reported one or more execution errors. */
361
+ withErrors: number;
362
+ /** Runs that explicitly reported zero execution errors. */
363
+ withoutErrors: number;
364
+ /** Runs with no execution-error count from the producer. */
365
+ unreported: number;
366
+ }
299
367
  interface TokenUsageInsight {
300
368
  input: ScalarDistribution;
301
369
  output: ScalarDistribution;
@@ -314,12 +382,18 @@ interface TokenUsageInsight {
314
382
  interface ScalarDistribution {
315
383
  /** Sample count after dropping non-finite values. */
316
384
  n: number;
317
- mean: number;
318
- p50: number;
319
- p95: number;
320
- stddev: number;
321
- min: number;
322
- max: number;
385
+ /** Null when `n` is zero. */
386
+ mean: number | null;
387
+ /** Null when `n` is zero. */
388
+ p50: number | null;
389
+ /** Null when `n` is zero. */
390
+ p95: number | null;
391
+ /** Null when `n` is zero. */
392
+ stddev: number | null;
393
+ /** Null when `n` is zero. */
394
+ min: number | null;
395
+ /** Null when `n` is zero. */
396
+ max: number | null;
323
397
  /** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
324
398
  histogram: GainDistributionBin[];
325
399
  /** Worst-N runs by score, ascending. Populated for the composite
@@ -385,12 +459,16 @@ interface LiftInsight {
385
459
  pValue: number;
386
460
  /** Number of paired observations. */
387
461
  n: number;
388
- /** Cohen's d for the delta. */
389
- cohensD: number;
462
+ /** Scored baseline observations without a candidate match. */
463
+ unpairedBaseline: number;
464
+ /** Scored candidate observations without a baseline match. */
465
+ unpairedCandidate: number;
466
+ /** Cohen's dz for paired deltas; null when the observed delta variance is zero. */
467
+ cohensD: number | null;
390
468
  /** Minimum detectable effect at current n, 80% power. */
391
469
  mde: number;
392
- /** Sample size needed to detect the observed delta at 80% power. */
393
- requiredN: number;
470
+ /** Paired sample size needed to detect the standardized effect at 80% power. */
471
+ requiredN: number | null;
394
472
  }
395
473
  interface FailureClusterInsight {
396
474
  /** All clusters identified by the registry, ranked by share descending. */
@@ -406,15 +484,13 @@ interface FailureClusterInsight {
406
484
  }>;
407
485
  totalFailures: number;
408
486
  }
409
- /** Model-free failure breakdown over the structured `RunRecord.failureMode`
410
- * enum. Unlike `failureClusters` (semantic, requires an LLM analyst), this
411
- * is computed directly from the tags the harness already recorded — so a
412
- * customer ingesting one batch with no judge/analyst still learns which
413
- * named failure dominates. */
414
- interface FailureModeTally {
415
- /** The `failureMode` tag. */
416
- mode: string;
417
- /** Number of runs carrying this tag. */
487
+ /** Model-free task-failure breakdown over canonical `RunRecord.failureClass`
488
+ * values. Unlike semantic failure clusters, this is computed directly from
489
+ * run records and does not require a model analyst. */
490
+ interface FailureClassTally {
491
+ /** Canonical task-failure class. */
492
+ failureClass: FailureClass;
493
+ /** Number of failed runs carrying this class. */
418
494
  count: number;
419
495
  /** Share of the whole corpus, 0..1. */
420
496
  share: number;
@@ -454,7 +530,7 @@ interface ReleaseSummary {
454
530
  status: 'pass' | 'warn' | 'fail';
455
531
  axes: Array<{
456
532
  name: 'quality-lift' | 'contamination' | 'composite-distribution';
457
- status: 'pass' | 'warn' | 'fail';
533
+ status: 'pass' | 'warn' | 'fail' | 'not_evaluated';
458
534
  detail: string;
459
535
  }>;
460
536
  /** Free-form issues surfaced beyond the standard axes. Empty by default;
@@ -514,9 +590,8 @@ interface Recommendation {
514
590
  * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
515
591
  * a partner's self-hosted one, a future open implementation) must accept.
516
592
  *
517
- * **Stability:** every type in this file is committed under semver. New
518
- * minors only ADD optional fields. Breaking changes mean a major bump
519
- * (`HostedWireVersion` literal increment).
593
+ * This package implements exactly one wire version. Servers reject every
594
+ * other version instead of translating old payloads.
520
595
  *
521
596
  * The wire format is two event streams in one transport:
522
597
  *
@@ -535,7 +610,7 @@ interface Recommendation {
535
610
  * ever sees another tenant's data.
536
611
  */
537
612
 
538
- declare const HOSTED_WIRE_VERSION: "2026-05-26.v1";
613
+ declare const HOSTED_WIRE_VERSION: "2026-07-24.v1";
539
614
  type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
540
615
  /** Every ingest request carries these. */
541
616
  interface HostedIngestHeaders {
@@ -545,8 +620,8 @@ interface HostedIngestHeaders {
545
620
  'x-tangle-tenant-id': string;
546
621
  /** Wire-version pin so the server can reject incompatible payloads. */
547
622
  'x-tangle-wire-version': HostedWireVersion;
548
- /** Optional idempotency key for retry-safe ingest. */
549
- 'idempotency-key'?: string;
623
+ /** Stable request key generated once and reused across retries. */
624
+ 'idempotency-key': string;
550
625
  }
551
626
  /** Lifecycle stages of an eval-run as the substrate reports them. */
552
627
  type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
@@ -555,11 +630,15 @@ interface EvalRunCellScore {
555
630
  scenarioId: string;
556
631
  /** Repetition index when reps > 1; 0 for the default. */
557
632
  rep: number;
558
- /** Composite score across all judges + dimensions for this cell. */
559
- compositeMean: number;
560
- /** Per-judge per-dimension scores; null where the judge did not run. */
633
+ /** Composite score across successful judges, or null when unscored. */
634
+ compositeMean: number | null;
635
+ /** Per-judge and per-dimension scores; failed or missing judges are absent. */
561
636
  dimensions: Record<string, Record<string, number>>;
562
- /** Per-cell error message if the dispatch threw. Null on success. */
637
+ /** Root execution result, kept separate from task quality. */
638
+ terminalOutcome: RunTerminalOutcome;
639
+ /** Canonical execution-error count, or null when the producer did not measure it. */
640
+ executionErrorCount: number | null;
641
+ /** Per-cell dispatch or judge error. Missing on success. */
563
642
  errorMessage?: string;
564
643
  }
565
644
  interface EvalRunGenerationSnapshot {
@@ -573,8 +652,8 @@ interface EvalRunGenerationSnapshot {
573
652
  surface?: MutableSurface;
574
653
  /** Per-cell scores for this generation. */
575
654
  cells: EvalRunCellScore[];
576
- /** Aggregate composite mean across all cells in this generation. */
577
- compositeMean: number;
655
+ /** Mean across scored cells, or null when no cell has a task-quality label. */
656
+ compositeMean: number | null;
578
657
  /** Total $ spent across this generation. */
579
658
  costUsd: number;
580
659
  /** Wall-clock duration of this generation. */
@@ -614,10 +693,14 @@ interface EvalRunEvent {
614
693
  * paired-bootstrap lift CI, judge stats, inter-rater agreement,
615
694
  * contamination check, failure clusters (when an analyst is wired),
616
695
  * outcome correlation (when downstream signal is supplied), and the
617
- * recommendations the dashboard surfaces verbatim. Additive; older
618
- * clients that don't know about this field continue to work. */
696
+ * recommendations the dashboard surfaces verbatim. */
619
697
  insightReport?: InsightReport;
620
698
  }
699
+ /**
700
+ * Canonical unsigned 64-bit integer encoded as a base-10 string.
701
+ * JSON numbers cannot represent OTLP nanosecond timestamps exactly.
702
+ */
703
+ type UnixNanoTimestamp = string;
621
704
  /**
622
705
  * OTel-shape span with a few additional attributes for eval-run pivoting.
623
706
  * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
@@ -628,11 +711,11 @@ interface TraceSpanEvent {
628
711
  spanId: string;
629
712
  parentSpanId?: string;
630
713
  name: string;
631
- startTimeUnixNano: number;
632
- endTimeUnixNano: number;
714
+ startTimeUnixNano: UnixNanoTimestamp;
715
+ endTimeUnixNano: UnixNanoTimestamp;
633
716
  attributes: Record<string, string | number | boolean>;
634
717
  events?: Array<{
635
- timeUnixNano: number;
718
+ timeUnixNano: UnixNanoTimestamp;
636
719
  name: string;
637
720
  attributes?: Record<string, string | number | boolean>;
638
721
  }>;
@@ -740,4 +823,22 @@ declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
740
823
  env?: Record<string, string | undefined>;
741
824
  }): HostedClient | undefined;
742
825
 
743
- export { type EvalRunCellScore, type EvalRunEvent, type EvalRunGenerationSnapshot, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, type IngestResponse, type IngestTracesRequest, type TraceSpanEvent, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };
826
+ declare const UnixNanoTimestampSchema: z.ZodType<UnixNanoTimestamp>;
827
+ declare const InsightReportSchema: z.ZodType<InsightReport>;
828
+ declare const MutableSurfaceSchema: z.ZodType<MutableSurface>;
829
+ declare const RunTerminalOutcomeSchema: z.ZodEnum<{
830
+ succeeded: "succeeded";
831
+ unknown: "unknown";
832
+ failed: "failed";
833
+ cancelled: "cancelled";
834
+ incomplete: "incomplete";
835
+ }>;
836
+ declare const EvalRunCellScoreSchema: z.ZodType<EvalRunCellScore>;
837
+ declare const EvalRunGenerationSnapshotSchema: z.ZodType<EvalRunGenerationSnapshot>;
838
+ declare const EvalRunEventSchema: z.ZodType<EvalRunEvent>;
839
+ declare const TraceSpanEventSchema: z.ZodType<TraceSpanEvent>;
840
+ declare const IngestEvalRunsRequestSchema: z.ZodType<IngestEvalRunsRequest>;
841
+ declare const IngestTracesRequestSchema: z.ZodType<IngestTracesRequest>;
842
+ declare const IngestResponseSchema: z.ZodType<IngestResponse>;
843
+
844
+ export { type EvalRunCellScore, EvalRunCellScoreSchema, type EvalRunEvent, EvalRunEventSchema, type EvalRunGenerationSnapshot, EvalRunGenerationSnapshotSchema, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, IngestEvalRunsRequestSchema, type IngestResponse, IngestResponseSchema, type IngestTracesRequest, IngestTracesRequestSchema, type InsightReport, InsightReportSchema, MutableSurfaceSchema, RunTerminalOutcomeSchema, type TraceSpanEvent, TraceSpanEventSchema, type UnixNanoTimestamp, UnixNanoTimestampSchema, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };
@@ -1,12 +1,35 @@
1
1
  import {
2
+ EvalRunCellScoreSchema,
3
+ EvalRunEventSchema,
4
+ EvalRunGenerationSnapshotSchema,
2
5
  HOSTED_WIRE_VERSION,
6
+ IngestEvalRunsRequestSchema,
7
+ IngestResponseSchema,
8
+ IngestTracesRequestSchema,
9
+ InsightReportSchema,
10
+ MutableSurfaceSchema,
11
+ RunTerminalOutcomeSchema,
12
+ TraceSpanEventSchema,
13
+ UnixNanoTimestampSchema,
3
14
  createHostedClient,
4
15
  hostedClientFromEnv,
5
16
  hostedTenantFromEnv
6
- } from "../chunk-ZZUXHH3R.js";
17
+ } from "../chunk-DRYIUNWY.js";
18
+ import "../chunk-MA6HLL3S.js";
7
19
  import "../chunk-PZ5AY32C.js";
8
20
  export {
21
+ EvalRunCellScoreSchema,
22
+ EvalRunEventSchema,
23
+ EvalRunGenerationSnapshotSchema,
9
24
  HOSTED_WIRE_VERSION,
25
+ IngestEvalRunsRequestSchema,
26
+ IngestResponseSchema,
27
+ IngestTracesRequestSchema,
28
+ InsightReportSchema,
29
+ MutableSurfaceSchema,
30
+ RunTerminalOutcomeSchema,
31
+ TraceSpanEventSchema,
32
+ UnixNanoTimestampSchema,
10
33
  createHostedClient,
11
34
  hostedClientFromEnv,
12
35
  hostedTenantFromEnv