@tangle-network/agent-eval 0.126.7 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +2 -2
- package/dist/analyst/index.d.ts +51 -39
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +34 -36
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +432 -146
- package/dist/campaign/index.js +11 -9
- package/dist/{chunk-ARU2PZFM.js → chunk-424XHVKV.js} +43 -23
- package/dist/chunk-424XHVKV.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-BTACVO2W.js → chunk-65QETEKG.js} +542 -258
- package/dist/chunk-65QETEKG.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-ARCFIY7U.js} +2 -2
- package/dist/{chunk-P22LJ3Y2.js → chunk-CLELFRQG.js} +6 -4
- package/dist/chunk-CLELFRQG.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-FA3MFYNA.js} +132 -110
- package/dist/chunk-FA3MFYNA.js.map +1 -0
- package/dist/chunk-IG35YDN3.js +614 -0
- package/dist/chunk-IG35YDN3.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-IMGJZBFG.js} +84 -56
- package/dist/chunk-IMGJZBFG.js.map +1 -0
- package/dist/{chunk-H5UD2323.js → chunk-IN455XKS.js} +53 -1
- package/dist/chunk-IN455XKS.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-IILEIWGW.js → chunk-NL33OXEP.js} +41 -36
- package/dist/chunk-NL33OXEP.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-OD5NQ2CS.js} +41 -16
- package/dist/chunk-OD5NQ2CS.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js} +50 -224
- package/dist/chunk-PRB53E3J.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-M7AH34KV.js → chunk-UTA2TGC4.js} +32 -16
- package/dist/chunk-UTA2TGC4.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-WED5KLZQ.js} +39 -15
- package/dist/chunk-WED5KLZQ.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-X6VOPMLD.js} +74 -14
- package/dist/chunk-X6VOPMLD.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-Y5CYOL42.js} +159 -70
- package/dist/chunk-Y5CYOL42.js.map +1 -0
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/chunk-ZPUEEM5U.js +179 -0
- package/dist/chunk-ZPUEEM5U.js.map +1 -0
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +421 -124
- package/dist/contract/index.js +242 -115
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +34 -32
- package/dist/control.js +2 -2
- package/dist/hosted/index.d.ts +142 -41
- package/dist/hosted/index.js +23 -1
- package/dist/index.d.ts +364 -255
- package/dist/index.js +315 -226
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +32 -32
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +97 -74
- package/dist/reporting.js +5 -4
- package/dist/rl.d.ts +177 -131
- package/dist/rl.js +232 -126
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +53 -63
- package/dist/rollout/index.js +7 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-3A2EBYBE.js} +4 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +66 -41
- package/dist/traces.js +9 -5
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +57 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-ARCFIY7U.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-3A2EBYBE.js.map} +0 -0
|
@@ -420,7 +420,8 @@ interface AgentProfileCell {
|
|
|
420
420
|
* researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
|
|
421
421
|
* fields are exactly those the paper "Two Loops, Three Roles" requires
|
|
422
422
|
* for reproducibility: who/what/when/cost/seed/hash, plus the search vs
|
|
423
|
-
* holdout split tag
|
|
423
|
+
* holdout split tag. A task score is optional because execution-only records
|
|
424
|
+
* must preserve missing labels instead of converting errors into zero quality.
|
|
424
425
|
*
|
|
425
426
|
* This is intentionally NOT a replacement for the rich `Run` /
|
|
426
427
|
* `ProposeReviewReport` / `ScenarioResult` types already in the
|
|
@@ -441,6 +442,13 @@ interface AgentProfileCell {
|
|
|
441
442
|
/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
|
|
442
443
|
* combined train+test pool that the optimizer is allowed to read. */
|
|
443
444
|
type RunSplitTag = 'search' | 'dev' | 'holdout';
|
|
445
|
+
/**
|
|
446
|
+
* Explicit execution-lifecycle result for a run.
|
|
447
|
+
*
|
|
448
|
+
* This is separate from task quality (`outcome`) and failure classification.
|
|
449
|
+
* Producers set it only from root-run or process evidence.
|
|
450
|
+
*/
|
|
451
|
+
type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
|
|
444
452
|
interface RunTokenUsage {
|
|
445
453
|
input: number;
|
|
446
454
|
/** All generated tokens charged as output, including reasoning tokens. */
|
|
@@ -454,11 +462,6 @@ interface RunTokenUsage {
|
|
|
454
462
|
}
|
|
455
463
|
/**
|
|
456
464
|
* How a run's USD amount was obtained.
|
|
457
|
-
*
|
|
458
|
-
* `costUsd` remains mandatory for wire compatibility. New producers should
|
|
459
|
-
* always populate this discriminated union so a missing bill is never
|
|
460
|
-
* mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
|
|
461
|
-
* the legacy `0` sentinel while this field carries the truthful null.
|
|
462
465
|
*/
|
|
463
466
|
type RunCostProvenance = {
|
|
464
467
|
kind: 'observed';
|
|
@@ -504,8 +507,8 @@ interface JudgeScoresRecord {
|
|
|
504
507
|
perJudge: Record<string, Record<string, number>>;
|
|
505
508
|
/** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
|
|
506
509
|
perDimMean: Record<string, number>;
|
|
507
|
-
/** Composite mean across
|
|
508
|
-
*
|
|
510
|
+
/** Composite mean across successful judges. Mirrors the task score only
|
|
511
|
+
* when `failedJudges` is empty. */
|
|
509
512
|
composite: number;
|
|
510
513
|
/** Judges that errored or returned an unparseable verdict. Recorded
|
|
511
514
|
* by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
|
|
@@ -516,11 +519,11 @@ interface JudgeScoresRecord {
|
|
|
516
519
|
notes?: string;
|
|
517
520
|
}
|
|
518
521
|
interface RunOutcome {
|
|
519
|
-
/** Score on the search/optimization split. Optional
|
|
520
|
-
*
|
|
522
|
+
/** Score on the search/optimization split. Optional for holdout-only and
|
|
523
|
+
* execution-only records. */
|
|
521
524
|
searchScore?: number;
|
|
522
|
-
/** Score on the held-out split. Optional
|
|
523
|
-
*
|
|
525
|
+
/** Score on the held-out split. Optional for search-only and execution-only
|
|
526
|
+
* records. When both scores are absent, the run is explicitly unlabeled. */
|
|
524
527
|
holdoutScore?: number;
|
|
525
528
|
/** Bag of any other metric the run produced — judge dimensions,
|
|
526
529
|
* pass/fail counters, latency stats, etc. Numeric only — keeps
|
|
@@ -584,42 +587,39 @@ interface RunRecord {
|
|
|
584
587
|
wallMs: number;
|
|
585
588
|
/** Time spent queued before execution started, if known. */
|
|
586
589
|
queueMs?: number;
|
|
587
|
-
/** Total USD cost
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
costUsd: number;
|
|
592
|
-
/** Observed, model-priced estimate, or genuinely uncaptured USD amount.
|
|
593
|
-
* Optional only so existing serialized RunRecords remain valid. */
|
|
594
|
-
costProvenance?: RunCostProvenance;
|
|
590
|
+
/** Total USD cost, or null when the producer could not capture one. */
|
|
591
|
+
costUsd: number | null;
|
|
592
|
+
/** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
|
|
593
|
+
costProvenance: RunCostProvenance;
|
|
595
594
|
/** Token usage breakdown. */
|
|
596
595
|
tokenUsage: RunTokenUsage;
|
|
596
|
+
/** Root-run or process terminal result. Never inferred from a child span. */
|
|
597
|
+
terminalOutcome: RunTerminalOutcome;
|
|
598
|
+
/** Root-run or process failure reason. Valid only for a failed, cancelled,
|
|
599
|
+
* or incomplete terminal result; never populated from a child span. */
|
|
600
|
+
terminalFailureReason?: string;
|
|
597
601
|
/** Judge-side metadata, if a judge was used. */
|
|
598
602
|
judgeMetadata?: RunJudgeMetadata;
|
|
599
603
|
/** Per-split scores + raw bag. */
|
|
600
604
|
outcome: RunOutcome;
|
|
601
|
-
/** Canonical
|
|
605
|
+
/** Canonical task-failure class drawn from the shared
|
|
602
606
|
* `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
|
|
603
607
|
* "which failure dominates across the whole fleet" answerable in ONE
|
|
604
608
|
* vocabulary — every agent classifies against the same enum. Producers
|
|
605
|
-
* set it
|
|
606
|
-
*
|
|
609
|
+
* set it only from task-result evidence. Execution errors belong in
|
|
610
|
+
* `outcome.raw.execution_error_count`, even when the run later fails. */
|
|
607
611
|
failureClass?: FailureClass;
|
|
608
|
-
/** Free-form
|
|
612
|
+
/** Free-form task-failure detail, scoped UNDER `failureClass`
|
|
609
613
|
* (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
|
|
610
|
-
*
|
|
614
|
+
* Do not populate this from a child execution error alone. */
|
|
611
615
|
failureMode?: string;
|
|
612
616
|
/** Which split this run was drawn from. */
|
|
613
617
|
splitTag: RunSplitTag;
|
|
614
618
|
/**
|
|
615
|
-
* Stable scenario identifier the run was scored against.
|
|
616
|
-
*
|
|
617
|
-
* that pairs runs by scenario (preferences, paired stats, BT tournament)
|
|
618
|
-
* keys on this. The campaign artifact populates it canonically; legacy
|
|
619
|
-
* runs without it fall back to inference from `outcome.raw.scenario_id`
|
|
620
|
-
* or `experimentId`.
|
|
619
|
+
* Stable scenario identifier the run observed or was scored against.
|
|
620
|
+
* Comparison primitives match this identity rather than input order.
|
|
621
621
|
*/
|
|
622
|
-
scenarioId
|
|
622
|
+
scenarioId: string;
|
|
623
623
|
/**
|
|
624
624
|
* Canonical identity for the agent profile cell that produced this row:
|
|
625
625
|
* profile artifact hash plus optional harness/model/prompt/reporting
|
package/dist/meta-eval/index.js
CHANGED
|
@@ -11,11 +11,11 @@ import {
|
|
|
11
11
|
} from "../chunk-3RF76KTD.js";
|
|
12
12
|
import {
|
|
13
13
|
rubricPredictiveValidity
|
|
14
|
-
} from "../chunk-
|
|
14
|
+
} from "../chunk-S5YLIBFX.js";
|
|
15
15
|
import {
|
|
16
16
|
pearsonR,
|
|
17
17
|
spearmanR
|
|
18
|
-
} from "../chunk-
|
|
18
|
+
} from "../chunk-MHELPNRP.js";
|
|
19
19
|
import {
|
|
20
20
|
aggregateLlm,
|
|
21
21
|
llmSpans
|
|
@@ -176,6 +176,7 @@ declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToo
|
|
|
176
176
|
* OpenAI-compatible `/v1/chat/completions` client with:
|
|
177
177
|
* - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
|
|
178
178
|
* - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
|
|
179
|
+
* - One retry at temperature 1 when a model explicitly requires it.
|
|
179
180
|
* - Graceful json_schema → json_object degrade on 400 with schema-reject body.
|
|
180
181
|
* - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
|
|
181
182
|
* - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.127.0",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
package/dist/pipelines/index.js
CHANGED
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
classifyFailure,
|
|
4
4
|
compareToBaseline,
|
|
5
5
|
computeToolUseMetrics
|
|
6
|
-
} from "../chunk-
|
|
6
|
+
} from "../chunk-P5W7RQKK.js";
|
|
7
7
|
import {
|
|
8
8
|
buildTrajectory
|
|
9
9
|
} from "../chunk-RZTMDUO7.js";
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
import {
|
|
14
14
|
interRaterReliability,
|
|
15
15
|
pearsonR
|
|
16
|
-
} from "../chunk-
|
|
16
|
+
} from "../chunk-MHELPNRP.js";
|
|
17
17
|
import {
|
|
18
18
|
aggregateLlm,
|
|
19
19
|
argHash,
|
package/dist/reporting.d.ts
CHANGED
|
@@ -31,7 +31,8 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
|
|
|
31
31
|
* researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
|
|
32
32
|
* fields are exactly those the paper "Two Loops, Three Roles" requires
|
|
33
33
|
* for reproducibility: who/what/when/cost/seed/hash, plus the search vs
|
|
34
|
-
* holdout split tag
|
|
34
|
+
* holdout split tag. A task score is optional because execution-only records
|
|
35
|
+
* must preserve missing labels instead of converting errors into zero quality.
|
|
35
36
|
*
|
|
36
37
|
* This is intentionally NOT a replacement for the rich `Run` /
|
|
37
38
|
* `ProposeReviewReport` / `ScenarioResult` types already in the
|
|
@@ -52,6 +53,13 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
|
|
|
52
53
|
/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
|
|
53
54
|
* combined train+test pool that the optimizer is allowed to read. */
|
|
54
55
|
type RunSplitTag = 'search' | 'dev' | 'holdout';
|
|
56
|
+
/**
|
|
57
|
+
* Explicit execution-lifecycle result for a run.
|
|
58
|
+
*
|
|
59
|
+
* This is separate from task quality (`outcome`) and failure classification.
|
|
60
|
+
* Producers set it only from root-run or process evidence.
|
|
61
|
+
*/
|
|
62
|
+
type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
|
|
55
63
|
interface RunTokenUsage {
|
|
56
64
|
input: number;
|
|
57
65
|
/** All generated tokens charged as output, including reasoning tokens. */
|
|
@@ -65,11 +73,6 @@ interface RunTokenUsage {
|
|
|
65
73
|
}
|
|
66
74
|
/**
|
|
67
75
|
* How a run's USD amount was obtained.
|
|
68
|
-
*
|
|
69
|
-
* `costUsd` remains mandatory for wire compatibility. New producers should
|
|
70
|
-
* always populate this discriminated union so a missing bill is never
|
|
71
|
-
* mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
|
|
72
|
-
* the legacy `0` sentinel while this field carries the truthful null.
|
|
73
76
|
*/
|
|
74
77
|
type RunCostProvenance = {
|
|
75
78
|
kind: 'observed';
|
|
@@ -115,8 +118,8 @@ interface JudgeScoresRecord {
|
|
|
115
118
|
perJudge: Record<string, Record<string, number>>;
|
|
116
119
|
/** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
|
|
117
120
|
perDimMean: Record<string, number>;
|
|
118
|
-
/** Composite mean across
|
|
119
|
-
*
|
|
121
|
+
/** Composite mean across successful judges. Mirrors the task score only
|
|
122
|
+
* when `failedJudges` is empty. */
|
|
120
123
|
composite: number;
|
|
121
124
|
/** Judges that errored or returned an unparseable verdict. Recorded
|
|
122
125
|
* by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
|
|
@@ -127,11 +130,11 @@ interface JudgeScoresRecord {
|
|
|
127
130
|
notes?: string;
|
|
128
131
|
}
|
|
129
132
|
interface RunOutcome {
|
|
130
|
-
/** Score on the search/optimization split. Optional
|
|
131
|
-
*
|
|
133
|
+
/** Score on the search/optimization split. Optional for holdout-only and
|
|
134
|
+
* execution-only records. */
|
|
132
135
|
searchScore?: number;
|
|
133
|
-
/** Score on the held-out split. Optional
|
|
134
|
-
*
|
|
136
|
+
/** Score on the held-out split. Optional for search-only and execution-only
|
|
137
|
+
* records. When both scores are absent, the run is explicitly unlabeled. */
|
|
135
138
|
holdoutScore?: number;
|
|
136
139
|
/** Bag of any other metric the run produced — judge dimensions,
|
|
137
140
|
* pass/fail counters, latency stats, etc. Numeric only — keeps
|
|
@@ -195,42 +198,39 @@ interface RunRecord {
|
|
|
195
198
|
wallMs: number;
|
|
196
199
|
/** Time spent queued before execution started, if known. */
|
|
197
200
|
queueMs?: number;
|
|
198
|
-
/** Total USD cost
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
costUsd: number;
|
|
203
|
-
/** Observed, model-priced estimate, or genuinely uncaptured USD amount.
|
|
204
|
-
* Optional only so existing serialized RunRecords remain valid. */
|
|
205
|
-
costProvenance?: RunCostProvenance;
|
|
201
|
+
/** Total USD cost, or null when the producer could not capture one. */
|
|
202
|
+
costUsd: number | null;
|
|
203
|
+
/** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
|
|
204
|
+
costProvenance: RunCostProvenance;
|
|
206
205
|
/** Token usage breakdown. */
|
|
207
206
|
tokenUsage: RunTokenUsage;
|
|
207
|
+
/** Root-run or process terminal result. Never inferred from a child span. */
|
|
208
|
+
terminalOutcome: RunTerminalOutcome;
|
|
209
|
+
/** Root-run or process failure reason. Valid only for a failed, cancelled,
|
|
210
|
+
* or incomplete terminal result; never populated from a child span. */
|
|
211
|
+
terminalFailureReason?: string;
|
|
208
212
|
/** Judge-side metadata, if a judge was used. */
|
|
209
213
|
judgeMetadata?: RunJudgeMetadata;
|
|
210
214
|
/** Per-split scores + raw bag. */
|
|
211
215
|
outcome: RunOutcome;
|
|
212
|
-
/** Canonical
|
|
216
|
+
/** Canonical task-failure class drawn from the shared
|
|
213
217
|
* `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
|
|
214
218
|
* "which failure dominates across the whole fleet" answerable in ONE
|
|
215
219
|
* vocabulary — every agent classifies against the same enum. Producers
|
|
216
|
-
* set it
|
|
217
|
-
*
|
|
220
|
+
* set it only from task-result evidence. Execution errors belong in
|
|
221
|
+
* `outcome.raw.execution_error_count`, even when the run later fails. */
|
|
218
222
|
failureClass?: FailureClass;
|
|
219
|
-
/** Free-form
|
|
223
|
+
/** Free-form task-failure detail, scoped UNDER `failureClass`
|
|
220
224
|
* (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
|
|
221
|
-
*
|
|
225
|
+
* Do not populate this from a child execution error alone. */
|
|
222
226
|
failureMode?: string;
|
|
223
227
|
/** Which split this run was drawn from. */
|
|
224
228
|
splitTag: RunSplitTag;
|
|
225
229
|
/**
|
|
226
|
-
* Stable scenario identifier the run was scored against.
|
|
227
|
-
*
|
|
228
|
-
* that pairs runs by scenario (preferences, paired stats, BT tournament)
|
|
229
|
-
* keys on this. The campaign artifact populates it canonically; legacy
|
|
230
|
-
* runs without it fall back to inference from `outcome.raw.scenario_id`
|
|
231
|
-
* or `experimentId`.
|
|
230
|
+
* Stable scenario identifier the run observed or was scored against.
|
|
231
|
+
* Comparison primitives match this identity rather than input order.
|
|
232
232
|
*/
|
|
233
|
-
scenarioId
|
|
233
|
+
scenarioId: string;
|
|
234
234
|
/**
|
|
235
235
|
* Canonical identity for the agent profile cell that produced this row:
|
|
236
236
|
* profile artifact hash plus optional harness/model/prompt/reporting
|
|
@@ -568,35 +568,39 @@ interface DatasetManifest {
|
|
|
568
568
|
* specific promotion path (still useful for replay-style evals).
|
|
569
569
|
*/
|
|
570
570
|
|
|
571
|
-
type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
571
|
+
type HeldOutGateRejectionCode = 'few_runs' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
572
572
|
interface GateEvidence {
|
|
573
573
|
/** Number of paired (candidate, baseline) holdout observations used. */
|
|
574
574
|
productiveRuns: number;
|
|
575
|
-
/**
|
|
576
|
-
|
|
577
|
-
/**
|
|
575
|
+
/** Candidate holdout rows with no baseline row at the same work identity. */
|
|
576
|
+
unpairedCandidateRuns: number;
|
|
577
|
+
/** Baseline holdout rows with no candidate row at the same work identity. */
|
|
578
|
+
unpairedBaselineRuns: number;
|
|
579
|
+
/** Median of paired holdout deltas, or null when there are no pairs. */
|
|
580
|
+
medianPairedDelta: number | null;
|
|
581
|
+
/** Bootstrap CI on the median paired holdout delta, if computed. */
|
|
578
582
|
pairedCI: {
|
|
579
583
|
low: number;
|
|
580
584
|
high: number;
|
|
581
|
-
};
|
|
582
|
-
/** Wilcoxon signed-rank p-value
|
|
583
|
-
pairedPValue: number;
|
|
584
|
-
/** Mean candidate score on the search split
|
|
585
|
-
searchScore: number;
|
|
586
|
-
/** Mean candidate score on the holdout split
|
|
587
|
-
holdoutScore: number;
|
|
588
|
-
/** Candidate (search − holdout) gap. */
|
|
589
|
-
overfitGap: number;
|
|
590
|
-
/** Baseline (search − holdout) gap. */
|
|
591
|
-
baselineOverfitGap: number;
|
|
585
|
+
} | null;
|
|
586
|
+
/** Wilcoxon signed-rank p-value, if computed. */
|
|
587
|
+
pairedPValue: number | null;
|
|
588
|
+
/** Mean candidate score on the search split, or null when absent. */
|
|
589
|
+
searchScore: number | null;
|
|
590
|
+
/** Mean candidate score on the holdout split, or null when absent. */
|
|
591
|
+
holdoutScore: number | null;
|
|
592
|
+
/** Candidate (search − holdout) gap, or null when either side is absent. */
|
|
593
|
+
overfitGap: number | null;
|
|
594
|
+
/** Baseline (search − holdout) gap, or null when either side is absent. */
|
|
595
|
+
baselineOverfitGap: number | null;
|
|
592
596
|
/** Median per-task USD cost across the candidate's runs. Recorded
|
|
593
597
|
* even when no `costPerTaskCeiling` is configured so downstream
|
|
594
598
|
* dashboards (intelligence.tangle.tools) can render \$/task per
|
|
595
599
|
* generation regardless of gating policy. */
|
|
596
|
-
medianCandidateCost: number;
|
|
600
|
+
medianCandidateCost: number | null;
|
|
597
601
|
/** Median per-task USD cost across the baseline runs, for
|
|
598
602
|
* symmetric reporting. */
|
|
599
|
-
medianBaselineCost: number;
|
|
603
|
+
medianBaselineCost: number | null;
|
|
600
604
|
}
|
|
601
605
|
interface GateDecision {
|
|
602
606
|
/** Final promote/no-promote verdict. */
|
|
@@ -647,7 +651,7 @@ interface ActionableSideInfo {
|
|
|
647
651
|
metadata?: Record<string, unknown>;
|
|
648
652
|
}
|
|
649
653
|
type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
|
|
650
|
-
type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
|
|
654
|
+
type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'reliability' | 'generalization' | 'diagnostics' | 'efficiency';
|
|
651
655
|
interface ReleaseTraceEvidence {
|
|
652
656
|
scenarioId: string;
|
|
653
657
|
candidateId?: string;
|
|
@@ -694,7 +698,7 @@ interface ReleaseConfidenceInput {
|
|
|
694
698
|
interface ReleaseConfidenceAxis {
|
|
695
699
|
name: ReleaseConfidenceAxisName;
|
|
696
700
|
status: ReleaseConfidenceStatus;
|
|
697
|
-
score: number;
|
|
701
|
+
score: number | null;
|
|
698
702
|
detail: string;
|
|
699
703
|
}
|
|
700
704
|
interface ReleaseConfidenceIssue {
|
|
@@ -705,15 +709,25 @@ interface ReleaseConfidenceIssue {
|
|
|
705
709
|
}
|
|
706
710
|
interface ReleaseConfidenceMetrics {
|
|
707
711
|
scenarioCount: number;
|
|
712
|
+
/** Search rows with a finite search score. */
|
|
708
713
|
searchRuns: number;
|
|
714
|
+
/** Holdout rows with a finite holdout score. */
|
|
709
715
|
holdoutRuns: number;
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
716
|
+
/** Runs with neither a split-matched score nor an explicit task failure. */
|
|
717
|
+
unscoredRuns: number;
|
|
718
|
+
/** Run rows, or trace rows when no runs exist, with no classified terminal result. */
|
|
719
|
+
unclassifiedTerminalRuns: number;
|
|
720
|
+
/** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */
|
|
721
|
+
terminalFailureRuns: number;
|
|
722
|
+
/** Success fraction when every run or fallback trace row has a classified result. */
|
|
723
|
+
reliabilityRate: number | null;
|
|
724
|
+
passRate: number | null;
|
|
725
|
+
meanScore: number | null;
|
|
726
|
+
searchMeanScore: number | null;
|
|
727
|
+
holdoutMeanScore: number | null;
|
|
728
|
+
overfitGap: number | null;
|
|
729
|
+
meanCostUsd: number | null;
|
|
730
|
+
p95WallMs: number | null;
|
|
717
731
|
failedRows: number;
|
|
718
732
|
failuresWithAsi: number;
|
|
719
733
|
singleShotTraces: number;
|
|
@@ -994,7 +1008,7 @@ interface FailureClusterReport {
|
|
|
994
1008
|
*/
|
|
995
1009
|
|
|
996
1010
|
interface SummaryTableOptions {
|
|
997
|
-
/** Comparator candidate id. Wilcoxon + Cohen's
|
|
1011
|
+
/** Comparator candidate id. Wilcoxon + paired Cohen's dz are computed
|
|
998
1012
|
* versus this candidate. Required for paired stats columns. */
|
|
999
1013
|
comparator?: string;
|
|
1000
1014
|
/** Which split to read scores from. Default 'holdout'. */
|
|
@@ -1010,10 +1024,16 @@ interface SummaryTableRow {
|
|
|
1010
1024
|
mean: number;
|
|
1011
1025
|
ciLow: number;
|
|
1012
1026
|
ciHigh: number;
|
|
1013
|
-
/** BH-adjusted q-value vs comparator
|
|
1014
|
-
qValue: number;
|
|
1015
|
-
/** Cohen's
|
|
1016
|
-
cohensD: number;
|
|
1027
|
+
/** BH-adjusted q-value vs comparator, or null when unavailable. */
|
|
1028
|
+
qValue: number | null;
|
|
1029
|
+
/** Paired Cohen's dz vs comparator, or null when the paired variance is zero. */
|
|
1030
|
+
cohensD: number | null;
|
|
1031
|
+
/** Matched observations used for paired comparison, or null on the comparator row. */
|
|
1032
|
+
pairedN: number | null;
|
|
1033
|
+
/** Candidate observations without a comparator match. */
|
|
1034
|
+
unpairedCandidateN: number | null;
|
|
1035
|
+
/** Comparator observations without a candidate match. */
|
|
1036
|
+
unpairedComparatorN: number | null;
|
|
1017
1037
|
}
|
|
1018
1038
|
interface SummaryTable {
|
|
1019
1039
|
rows: SummaryTableRow[];
|
|
@@ -1025,7 +1045,7 @@ interface SummaryTable {
|
|
|
1025
1045
|
/**
|
|
1026
1046
|
* Table 1 helper. Buckets runs by `candidateId`, computes mean +
|
|
1027
1047
|
* bootstrap CI on the chosen split, and (when a comparator is given)
|
|
1028
|
-
* BH-adjusted Wilcoxon p + Cohen's
|
|
1048
|
+
* BH-adjusted Wilcoxon p + paired Cohen's dz versus that comparator.
|
|
1029
1049
|
*/
|
|
1030
1050
|
declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
|
|
1031
1051
|
interface ParetoPoint {
|
|
@@ -1041,7 +1061,7 @@ interface ParetoPoint {
|
|
|
1041
1061
|
onFrontier: boolean;
|
|
1042
1062
|
/** Optional gate verdict for this candidate, if a `GateDecision`
|
|
1043
1063
|
* for it was passed in. */
|
|
1044
|
-
gate?: 'promote' | '
|
|
1064
|
+
gate?: 'promote' | 'reject';
|
|
1045
1065
|
}
|
|
1046
1066
|
interface ParetoFigureSpec {
|
|
1047
1067
|
kind: 'pareto-cost-quality';
|
|
@@ -1076,12 +1096,16 @@ interface GainDistributionFigureSpec {
|
|
|
1076
1096
|
split: 'search' | 'holdout';
|
|
1077
1097
|
/** Number of pairs used. */
|
|
1078
1098
|
n: number;
|
|
1099
|
+
/** Candidate rows without a comparator match. */
|
|
1100
|
+
unpairedCandidateN: number;
|
|
1101
|
+
/** Comparator rows without a candidate match. */
|
|
1102
|
+
unpairedComparatorN: number;
|
|
1079
1103
|
bins: GainDistributionBin[];
|
|
1080
|
-
median: number;
|
|
1104
|
+
median: number | null;
|
|
1081
1105
|
ci: {
|
|
1082
1106
|
low: number;
|
|
1083
1107
|
high: number;
|
|
1084
|
-
};
|
|
1108
|
+
} | null;
|
|
1085
1109
|
}
|
|
1086
1110
|
interface GainDistributionOptions {
|
|
1087
1111
|
/** Number of histogram bins. Default 11 (so the centre is exact at 0). */
|
|
@@ -1178,8 +1202,8 @@ interface ResearchReportCandidate {
|
|
|
1178
1202
|
mean: number;
|
|
1179
1203
|
ciLow: number;
|
|
1180
1204
|
ciHigh: number;
|
|
1181
|
-
qValue: number;
|
|
1182
|
-
cohensD: number;
|
|
1205
|
+
qValue: number | null;
|
|
1206
|
+
cohensD: number | null;
|
|
1183
1207
|
meanDeltaVsComparator: number | null;
|
|
1184
1208
|
pairedN: number;
|
|
1185
1209
|
medianGain: number | null;
|
|
@@ -1189,10 +1213,9 @@ interface ResearchReportCandidate {
|
|
|
1189
1213
|
high: number;
|
|
1190
1214
|
} | null;
|
|
1191
1215
|
/**
|
|
1192
|
-
* Bayesian-bootstrap
|
|
1193
|
-
*
|
|
1194
|
-
*
|
|
1195
|
-
* matched pairs."
|
|
1216
|
+
* Bayesian-bootstrap posterior summaries on the paired mean delta.
|
|
1217
|
+
* Dirichlet(1, ..., 1) weights represent uncertainty over the empirical
|
|
1218
|
+
* distribution of matched deltas.
|
|
1196
1219
|
*/
|
|
1197
1220
|
prGreaterThanZero: number | null;
|
|
1198
1221
|
prInRope: number | null;
|
|
@@ -1260,7 +1283,7 @@ interface ResearchReport {
|
|
|
1260
1283
|
* - `summaryTable` marginal stats with BH-FDR-adjusted q-values
|
|
1261
1284
|
* - `paretoChart` cost-vs-quality frontier with gate overlay
|
|
1262
1285
|
* - `gainHistogram` per-candidate paired-delta distribution
|
|
1263
|
-
* - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
|
|
1286
|
+
* - paired posterior (this file): bootstrap CI on median, Bayesian-bootstrap Pr(Δ>0),
|
|
1264
1287
|
* Pr(Δ∈ROPE), MDE at the configured power
|
|
1265
1288
|
*
|
|
1266
1289
|
* Decisions are made on paired evidence — never on marginal means alone —
|
package/dist/reporting.js
CHANGED
|
@@ -4,10 +4,10 @@ import {
|
|
|
4
4
|
evaluateReleaseConfidence,
|
|
5
5
|
judgeReplayGate,
|
|
6
6
|
renderReleaseReport
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-Y5CYOL42.js";
|
|
8
8
|
import {
|
|
9
9
|
rubricPredictiveValidity
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-S5YLIBFX.js";
|
|
11
11
|
import {
|
|
12
12
|
evaluateInterimReleaseConfidence,
|
|
13
13
|
pairedEvalueSequence
|
|
@@ -18,12 +18,13 @@ import {
|
|
|
18
18
|
paretoChart,
|
|
19
19
|
researchReport,
|
|
20
20
|
summaryTable
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-IMGJZBFG.js";
|
|
22
|
+
import "./chunk-DPUHNQLN.js";
|
|
22
23
|
import {
|
|
23
24
|
benjaminiHochberg,
|
|
24
25
|
pairedBootstrap,
|
|
25
26
|
wilcoxonSignedRank
|
|
26
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-MHELPNRP.js";
|
|
27
28
|
import "./chunk-VSMTAMNK.js";
|
|
28
29
|
import "./chunk-ONWEPEDO.js";
|
|
29
30
|
import "./chunk-PZ5AY32C.js";
|