@tangle-network/agent-eval 0.126.7 → 0.128.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +6 -2
- package/dist/analyst/index.d.ts +52 -43
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +35 -40
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -11
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +441 -160
- package/dist/campaign/index.js +11 -10
- package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
- package/dist/chunk-2JX3CFMB.js.map +1 -0
- package/dist/chunk-2MKQIFS4.js +183 -0
- package/dist/chunk-2MKQIFS4.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
- package/dist/chunk-7ZZMD7UK.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
- package/dist/chunk-BYT7ELPS.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/chunk-DRYIUNWY.js +622 -0
- package/dist/chunk-DRYIUNWY.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
- package/dist/chunk-EOSZT7PL.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
- package/dist/chunk-EZJEIH2R.js.map +1 -0
- package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
- package/dist/chunk-IHQDPH7D.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
- package/dist/chunk-NACAGYSY.js.map +1 -0
- package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
- package/dist/chunk-NKAGIDE2.js.map +1 -0
- package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
- package/dist/chunk-NYLOYM6N.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
- package/dist/chunk-TBL77AUT.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
- package/dist/chunk-UB2LOJ6Q.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
- package/dist/chunk-VLOATJQ2.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
- package/dist/chunk-XDWDC2MP.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +438 -138
- package/dist/contract/index.js +259 -113
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +56 -39
- package/dist/control.js +2 -3
- package/dist/hosted/index.d.ts +152 -51
- package/dist/hosted/index.js +24 -1
- package/dist/index.d.ts +399 -277
- package/dist/index.js +363 -244
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +33 -36
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +101 -80
- package/dist/reporting.js +7 -5
- package/dist/rl.d.ts +198 -142
- package/dist/rl.js +235 -127
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +54 -67
- package/dist/rollout/index.js +6 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +72 -50
- package/dist/traces.js +9 -6
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/customer-journeys.md +7 -6
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +78 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-VSMTAMNK.js +0 -53
- package/dist/chunk-VSMTAMNK.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
|
@@ -420,7 +420,8 @@ interface AgentProfileCell {
|
|
|
420
420
|
* researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
|
|
421
421
|
* fields are exactly those the paper "Two Loops, Three Roles" requires
|
|
422
422
|
* for reproducibility: who/what/when/cost/seed/hash, plus the search vs
|
|
423
|
-
* holdout split tag
|
|
423
|
+
* holdout split tag. A task score is optional because execution-only records
|
|
424
|
+
* must preserve missing labels instead of converting errors into zero quality.
|
|
424
425
|
*
|
|
425
426
|
* This is intentionally NOT a replacement for the rich `Run` /
|
|
426
427
|
* `ProposeReviewReport` / `ScenarioResult` types already in the
|
|
@@ -441,6 +442,13 @@ interface AgentProfileCell {
|
|
|
441
442
|
/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
|
|
442
443
|
* combined train+test pool that the optimizer is allowed to read. */
|
|
443
444
|
type RunSplitTag = 'search' | 'dev' | 'holdout';
|
|
445
|
+
/**
|
|
446
|
+
* Explicit execution-lifecycle result for a run.
|
|
447
|
+
*
|
|
448
|
+
* This is separate from task quality (`outcome`) and failure classification.
|
|
449
|
+
* Producers set it only from root-run or process evidence.
|
|
450
|
+
*/
|
|
451
|
+
type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
|
|
444
452
|
interface RunTokenUsage {
|
|
445
453
|
input: number;
|
|
446
454
|
/** All generated tokens charged as output, including reasoning tokens. */
|
|
@@ -454,11 +462,6 @@ interface RunTokenUsage {
|
|
|
454
462
|
}
|
|
455
463
|
/**
|
|
456
464
|
* How a run's USD amount was obtained.
|
|
457
|
-
*
|
|
458
|
-
* `costUsd` remains mandatory for wire compatibility. New producers should
|
|
459
|
-
* always populate this discriminated union so a missing bill is never
|
|
460
|
-
* mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
|
|
461
|
-
* the legacy `0` sentinel while this field carries the truthful null.
|
|
462
465
|
*/
|
|
463
466
|
type RunCostProvenance = {
|
|
464
467
|
kind: 'observed';
|
|
@@ -504,8 +507,8 @@ interface JudgeScoresRecord {
|
|
|
504
507
|
perJudge: Record<string, Record<string, number>>;
|
|
505
508
|
/** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
|
|
506
509
|
perDimMean: Record<string, number>;
|
|
507
|
-
/** Composite mean across
|
|
508
|
-
*
|
|
510
|
+
/** Composite mean across successful judges. Mirrors the task score only
|
|
511
|
+
* when `failedJudges` is empty. */
|
|
509
512
|
composite: number;
|
|
510
513
|
/** Judges that errored or returned an unparseable verdict. Recorded
|
|
511
514
|
* by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
|
|
@@ -516,11 +519,11 @@ interface JudgeScoresRecord {
|
|
|
516
519
|
notes?: string;
|
|
517
520
|
}
|
|
518
521
|
interface RunOutcome {
|
|
519
|
-
/** Score on the search/optimization split. Optional
|
|
520
|
-
*
|
|
522
|
+
/** Score on the search/optimization split. Optional for holdout-only and
|
|
523
|
+
* execution-only records. */
|
|
521
524
|
searchScore?: number;
|
|
522
|
-
/** Score on the held-out split. Optional
|
|
523
|
-
*
|
|
525
|
+
/** Score on the held-out split. Optional for search-only and execution-only
|
|
526
|
+
* records. When both scores are absent, the run is explicitly unlabeled. */
|
|
524
527
|
holdoutScore?: number;
|
|
525
528
|
/** Bag of any other metric the run produced — judge dimensions,
|
|
526
529
|
* pass/fail counters, latency stats, etc. Numeric only — keeps
|
|
@@ -584,42 +587,36 @@ interface RunRecord {
|
|
|
584
587
|
wallMs: number;
|
|
585
588
|
/** Time spent queued before execution started, if known. */
|
|
586
589
|
queueMs?: number;
|
|
587
|
-
/** Total USD cost
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
costUsd: number;
|
|
592
|
-
/** Observed, model-priced estimate, or genuinely uncaptured USD amount.
|
|
593
|
-
* Optional only so existing serialized RunRecords remain valid. */
|
|
594
|
-
costProvenance?: RunCostProvenance;
|
|
590
|
+
/** Total USD cost, or null when the producer could not capture one. */
|
|
591
|
+
costUsd: number | null;
|
|
592
|
+
/** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
|
|
593
|
+
costProvenance: RunCostProvenance;
|
|
595
594
|
/** Token usage breakdown. */
|
|
596
595
|
tokenUsage: RunTokenUsage;
|
|
596
|
+
/** Root-run or process terminal result. Never inferred from a child span. */
|
|
597
|
+
terminalOutcome: RunTerminalOutcome;
|
|
598
|
+
/** Root-run or process failure reason. Valid only for a failed, cancelled,
|
|
599
|
+
* or incomplete terminal result; never populated from a child span. */
|
|
600
|
+
terminalFailureReason?: string;
|
|
597
601
|
/** Judge-side metadata, if a judge was used. */
|
|
598
602
|
judgeMetadata?: RunJudgeMetadata;
|
|
599
603
|
/** Per-split scores + raw bag. */
|
|
600
604
|
outcome: RunOutcome;
|
|
601
|
-
/** Canonical
|
|
602
|
-
* `FAILURE_CLASSES` taxonomy.
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
* set it via the substrate classifier; leave unset only when the failure
|
|
606
|
-
* genuinely can't be classified. */
|
|
605
|
+
/** Canonical task-failure class drawn from the shared
|
|
606
|
+
* `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
|
|
607
|
+
* evidence. Execution errors belong in
|
|
608
|
+
* `outcome.raw.execution_error_count`. */
|
|
607
609
|
failureClass?: FailureClass;
|
|
608
|
-
/** Free-form
|
|
609
|
-
*
|
|
610
|
-
* The within-agent drill-down; `failureClass` is the cross-agent key. */
|
|
610
|
+
/** Free-form task-failure detail scoped under a non-success
|
|
611
|
+
* `failureClass`. It is invalid without that class. */
|
|
611
612
|
failureMode?: string;
|
|
612
613
|
/** Which split this run was drawn from. */
|
|
613
614
|
splitTag: RunSplitTag;
|
|
614
615
|
/**
|
|
615
|
-
* Stable scenario identifier the run was scored against.
|
|
616
|
-
*
|
|
617
|
-
* that pairs runs by scenario (preferences, paired stats, BT tournament)
|
|
618
|
-
* keys on this. The campaign artifact populates it canonically; legacy
|
|
619
|
-
* runs without it fall back to inference from `outcome.raw.scenario_id`
|
|
620
|
-
* or `experimentId`.
|
|
616
|
+
* Stable scenario identifier the run observed or was scored against.
|
|
617
|
+
* Comparison primitives match this identity rather than input order.
|
|
621
618
|
*/
|
|
622
|
-
scenarioId
|
|
619
|
+
scenarioId: string;
|
|
623
620
|
/**
|
|
624
621
|
* Canonical identity for the agent profile cell that produced this row:
|
|
625
622
|
* profile artifact hash plus optional harness/model/prompt/reporting
|
package/dist/meta-eval/index.js
CHANGED
|
@@ -11,11 +11,11 @@ import {
|
|
|
11
11
|
} from "../chunk-3RF76KTD.js";
|
|
12
12
|
import {
|
|
13
13
|
rubricPredictiveValidity
|
|
14
|
-
} from "../chunk-
|
|
14
|
+
} from "../chunk-S5YLIBFX.js";
|
|
15
15
|
import {
|
|
16
16
|
pearsonR,
|
|
17
17
|
spearmanR
|
|
18
|
-
} from "../chunk-
|
|
18
|
+
} from "../chunk-MHELPNRP.js";
|
|
19
19
|
import {
|
|
20
20
|
aggregateLlm,
|
|
21
21
|
llmSpans
|
|
@@ -176,6 +176,7 @@ declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToo
|
|
|
176
176
|
* OpenAI-compatible `/v1/chat/completions` client with:
|
|
177
177
|
* - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
|
|
178
178
|
* - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
|
|
179
|
+
* - One retry at temperature 1 when a model explicitly requires it.
|
|
179
180
|
* - Graceful json_schema → json_object degrade on 400 with schema-reject body.
|
|
180
181
|
* - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
|
|
181
182
|
* - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.128.0",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
package/dist/pipelines/index.js
CHANGED
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
classifyFailure,
|
|
4
4
|
compareToBaseline,
|
|
5
5
|
computeToolUseMetrics
|
|
6
|
-
} from "../chunk-
|
|
6
|
+
} from "../chunk-P5W7RQKK.js";
|
|
7
7
|
import {
|
|
8
8
|
buildTrajectory
|
|
9
9
|
} from "../chunk-RZTMDUO7.js";
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
import {
|
|
14
14
|
interRaterReliability,
|
|
15
15
|
pearsonR
|
|
16
|
-
} from "../chunk-
|
|
16
|
+
} from "../chunk-MHELPNRP.js";
|
|
17
17
|
import {
|
|
18
18
|
aggregateLlm,
|
|
19
19
|
argHash,
|
package/dist/reporting.d.ts
CHANGED
|
@@ -31,7 +31,8 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
|
|
|
31
31
|
* researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
|
|
32
32
|
* fields are exactly those the paper "Two Loops, Three Roles" requires
|
|
33
33
|
* for reproducibility: who/what/when/cost/seed/hash, plus the search vs
|
|
34
|
-
* holdout split tag
|
|
34
|
+
* holdout split tag. A task score is optional because execution-only records
|
|
35
|
+
* must preserve missing labels instead of converting errors into zero quality.
|
|
35
36
|
*
|
|
36
37
|
* This is intentionally NOT a replacement for the rich `Run` /
|
|
37
38
|
* `ProposeReviewReport` / `ScenarioResult` types already in the
|
|
@@ -52,6 +53,13 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
|
|
|
52
53
|
/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
|
|
53
54
|
* combined train+test pool that the optimizer is allowed to read. */
|
|
54
55
|
type RunSplitTag = 'search' | 'dev' | 'holdout';
|
|
56
|
+
/**
|
|
57
|
+
* Explicit execution-lifecycle result for a run.
|
|
58
|
+
*
|
|
59
|
+
* This is separate from task quality (`outcome`) and failure classification.
|
|
60
|
+
* Producers set it only from root-run or process evidence.
|
|
61
|
+
*/
|
|
62
|
+
type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
|
|
55
63
|
interface RunTokenUsage {
|
|
56
64
|
input: number;
|
|
57
65
|
/** All generated tokens charged as output, including reasoning tokens. */
|
|
@@ -65,11 +73,6 @@ interface RunTokenUsage {
|
|
|
65
73
|
}
|
|
66
74
|
/**
|
|
67
75
|
* How a run's USD amount was obtained.
|
|
68
|
-
*
|
|
69
|
-
* `costUsd` remains mandatory for wire compatibility. New producers should
|
|
70
|
-
* always populate this discriminated union so a missing bill is never
|
|
71
|
-
* mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
|
|
72
|
-
* the legacy `0` sentinel while this field carries the truthful null.
|
|
73
76
|
*/
|
|
74
77
|
type RunCostProvenance = {
|
|
75
78
|
kind: 'observed';
|
|
@@ -115,8 +118,8 @@ interface JudgeScoresRecord {
|
|
|
115
118
|
perJudge: Record<string, Record<string, number>>;
|
|
116
119
|
/** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
|
|
117
120
|
perDimMean: Record<string, number>;
|
|
118
|
-
/** Composite mean across
|
|
119
|
-
*
|
|
121
|
+
/** Composite mean across successful judges. Mirrors the task score only
|
|
122
|
+
* when `failedJudges` is empty. */
|
|
120
123
|
composite: number;
|
|
121
124
|
/** Judges that errored or returned an unparseable verdict. Recorded
|
|
122
125
|
* by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
|
|
@@ -127,11 +130,11 @@ interface JudgeScoresRecord {
|
|
|
127
130
|
notes?: string;
|
|
128
131
|
}
|
|
129
132
|
interface RunOutcome {
|
|
130
|
-
/** Score on the search/optimization split. Optional
|
|
131
|
-
*
|
|
133
|
+
/** Score on the search/optimization split. Optional for holdout-only and
|
|
134
|
+
* execution-only records. */
|
|
132
135
|
searchScore?: number;
|
|
133
|
-
/** Score on the held-out split. Optional
|
|
134
|
-
*
|
|
136
|
+
/** Score on the held-out split. Optional for search-only and execution-only
|
|
137
|
+
* records. When both scores are absent, the run is explicitly unlabeled. */
|
|
135
138
|
holdoutScore?: number;
|
|
136
139
|
/** Bag of any other metric the run produced — judge dimensions,
|
|
137
140
|
* pass/fail counters, latency stats, etc. Numeric only — keeps
|
|
@@ -195,42 +198,36 @@ interface RunRecord {
|
|
|
195
198
|
wallMs: number;
|
|
196
199
|
/** Time spent queued before execution started, if known. */
|
|
197
200
|
queueMs?: number;
|
|
198
|
-
/** Total USD cost
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
costUsd: number;
|
|
203
|
-
/** Observed, model-priced estimate, or genuinely uncaptured USD amount.
|
|
204
|
-
* Optional only so existing serialized RunRecords remain valid. */
|
|
205
|
-
costProvenance?: RunCostProvenance;
|
|
201
|
+
/** Total USD cost, or null when the producer could not capture one. */
|
|
202
|
+
costUsd: number | null;
|
|
203
|
+
/** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
|
|
204
|
+
costProvenance: RunCostProvenance;
|
|
206
205
|
/** Token usage breakdown. */
|
|
207
206
|
tokenUsage: RunTokenUsage;
|
|
207
|
+
/** Root-run or process terminal result. Never inferred from a child span. */
|
|
208
|
+
terminalOutcome: RunTerminalOutcome;
|
|
209
|
+
/** Root-run or process failure reason. Valid only for a failed, cancelled,
|
|
210
|
+
* or incomplete terminal result; never populated from a child span. */
|
|
211
|
+
terminalFailureReason?: string;
|
|
208
212
|
/** Judge-side metadata, if a judge was used. */
|
|
209
213
|
judgeMetadata?: RunJudgeMetadata;
|
|
210
214
|
/** Per-split scores + raw bag. */
|
|
211
215
|
outcome: RunOutcome;
|
|
212
|
-
/** Canonical
|
|
213
|
-
* `FAILURE_CLASSES` taxonomy.
|
|
214
|
-
*
|
|
215
|
-
*
|
|
216
|
-
* set it via the substrate classifier; leave unset only when the failure
|
|
217
|
-
* genuinely can't be classified. */
|
|
216
|
+
/** Canonical task-failure class drawn from the shared
|
|
217
|
+
* `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
|
|
218
|
+
* evidence. Execution errors belong in
|
|
219
|
+
* `outcome.raw.execution_error_count`. */
|
|
218
220
|
failureClass?: FailureClass;
|
|
219
|
-
/** Free-form
|
|
220
|
-
*
|
|
221
|
-
* The within-agent drill-down; `failureClass` is the cross-agent key. */
|
|
221
|
+
/** Free-form task-failure detail scoped under a non-success
|
|
222
|
+
* `failureClass`. It is invalid without that class. */
|
|
222
223
|
failureMode?: string;
|
|
223
224
|
/** Which split this run was drawn from. */
|
|
224
225
|
splitTag: RunSplitTag;
|
|
225
226
|
/**
|
|
226
|
-
* Stable scenario identifier the run was scored against.
|
|
227
|
-
*
|
|
228
|
-
* that pairs runs by scenario (preferences, paired stats, BT tournament)
|
|
229
|
-
* keys on this. The campaign artifact populates it canonically; legacy
|
|
230
|
-
* runs without it fall back to inference from `outcome.raw.scenario_id`
|
|
231
|
-
* or `experimentId`.
|
|
227
|
+
* Stable scenario identifier the run observed or was scored against.
|
|
228
|
+
* Comparison primitives match this identity rather than input order.
|
|
232
229
|
*/
|
|
233
|
-
scenarioId
|
|
230
|
+
scenarioId: string;
|
|
234
231
|
/**
|
|
235
232
|
* Canonical identity for the agent profile cell that produced this row:
|
|
236
233
|
* profile artifact hash plus optional harness/model/prompt/reporting
|
|
@@ -568,35 +565,39 @@ interface DatasetManifest {
|
|
|
568
565
|
* specific promotion path (still useful for replay-style evals).
|
|
569
566
|
*/
|
|
570
567
|
|
|
571
|
-
type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
568
|
+
type HeldOutGateRejectionCode = 'few_runs' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
572
569
|
interface GateEvidence {
|
|
573
570
|
/** Number of paired (candidate, baseline) holdout observations used. */
|
|
574
571
|
productiveRuns: number;
|
|
575
|
-
/**
|
|
576
|
-
|
|
577
|
-
/**
|
|
572
|
+
/** Candidate holdout rows with no baseline row at the same work identity. */
|
|
573
|
+
unpairedCandidateRuns: number;
|
|
574
|
+
/** Baseline holdout rows with no candidate row at the same work identity. */
|
|
575
|
+
unpairedBaselineRuns: number;
|
|
576
|
+
/** Median of paired holdout deltas, or null when there are no pairs. */
|
|
577
|
+
medianPairedDelta: number | null;
|
|
578
|
+
/** Bootstrap CI on the median paired holdout delta, if computed. */
|
|
578
579
|
pairedCI: {
|
|
579
580
|
low: number;
|
|
580
581
|
high: number;
|
|
581
|
-
};
|
|
582
|
-
/** Wilcoxon signed-rank p-value
|
|
583
|
-
pairedPValue: number;
|
|
584
|
-
/** Mean candidate score on the search split
|
|
585
|
-
searchScore: number;
|
|
586
|
-
/** Mean candidate score on the holdout split
|
|
587
|
-
holdoutScore: number;
|
|
588
|
-
/** Candidate (search − holdout) gap. */
|
|
589
|
-
overfitGap: number;
|
|
590
|
-
/** Baseline (search − holdout) gap. */
|
|
591
|
-
baselineOverfitGap: number;
|
|
582
|
+
} | null;
|
|
583
|
+
/** Wilcoxon signed-rank p-value, if computed. */
|
|
584
|
+
pairedPValue: number | null;
|
|
585
|
+
/** Mean candidate score on the search split, or null when absent. */
|
|
586
|
+
searchScore: number | null;
|
|
587
|
+
/** Mean candidate score on the holdout split, or null when absent. */
|
|
588
|
+
holdoutScore: number | null;
|
|
589
|
+
/** Candidate (search − holdout) gap, or null when either side is absent. */
|
|
590
|
+
overfitGap: number | null;
|
|
591
|
+
/** Baseline (search − holdout) gap, or null when either side is absent. */
|
|
592
|
+
baselineOverfitGap: number | null;
|
|
592
593
|
/** Median per-task USD cost across the candidate's runs. Recorded
|
|
593
594
|
* even when no `costPerTaskCeiling` is configured so downstream
|
|
594
595
|
* dashboards (intelligence.tangle.tools) can render \$/task per
|
|
595
596
|
* generation regardless of gating policy. */
|
|
596
|
-
medianCandidateCost: number;
|
|
597
|
+
medianCandidateCost: number | null;
|
|
597
598
|
/** Median per-task USD cost across the baseline runs, for
|
|
598
599
|
* symmetric reporting. */
|
|
599
|
-
medianBaselineCost: number;
|
|
600
|
+
medianBaselineCost: number | null;
|
|
600
601
|
}
|
|
601
602
|
interface GateDecision {
|
|
602
603
|
/** Final promote/no-promote verdict. */
|
|
@@ -647,7 +648,7 @@ interface ActionableSideInfo {
|
|
|
647
648
|
metadata?: Record<string, unknown>;
|
|
648
649
|
}
|
|
649
650
|
type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
|
|
650
|
-
type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
|
|
651
|
+
type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'reliability' | 'generalization' | 'diagnostics' | 'efficiency';
|
|
651
652
|
interface ReleaseTraceEvidence {
|
|
652
653
|
scenarioId: string;
|
|
653
654
|
candidateId?: string;
|
|
@@ -657,7 +658,8 @@ interface ReleaseTraceEvidence {
|
|
|
657
658
|
turnCount?: number;
|
|
658
659
|
costUsd?: number;
|
|
659
660
|
durationMs?: number;
|
|
660
|
-
|
|
661
|
+
/** Canonical task-failure class. Free-form detail belongs in ASI. */
|
|
662
|
+
failureClass?: FailureClass;
|
|
661
663
|
asi?: ActionableSideInfo[];
|
|
662
664
|
metadata?: Record<string, unknown>;
|
|
663
665
|
}
|
|
@@ -694,7 +696,7 @@ interface ReleaseConfidenceInput {
|
|
|
694
696
|
interface ReleaseConfidenceAxis {
|
|
695
697
|
name: ReleaseConfidenceAxisName;
|
|
696
698
|
status: ReleaseConfidenceStatus;
|
|
697
|
-
score: number;
|
|
699
|
+
score: number | null;
|
|
698
700
|
detail: string;
|
|
699
701
|
}
|
|
700
702
|
interface ReleaseConfidenceIssue {
|
|
@@ -705,22 +707,32 @@ interface ReleaseConfidenceIssue {
|
|
|
705
707
|
}
|
|
706
708
|
interface ReleaseConfidenceMetrics {
|
|
707
709
|
scenarioCount: number;
|
|
710
|
+
/** Search rows with a finite search score. */
|
|
708
711
|
searchRuns: number;
|
|
712
|
+
/** Holdout rows with a finite holdout score. */
|
|
709
713
|
holdoutRuns: number;
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
714
|
+
/** Runs with neither a split-matched score nor an explicit task failure. */
|
|
715
|
+
unscoredRuns: number;
|
|
716
|
+
/** Run rows, or trace rows when no runs exist, with no classified terminal result. */
|
|
717
|
+
unclassifiedTerminalRuns: number;
|
|
718
|
+
/** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */
|
|
719
|
+
terminalFailureRuns: number;
|
|
720
|
+
/** Success fraction when every run or fallback trace row has a classified result. */
|
|
721
|
+
reliabilityRate: number | null;
|
|
722
|
+
passRate: number | null;
|
|
723
|
+
meanScore: number | null;
|
|
724
|
+
searchMeanScore: number | null;
|
|
725
|
+
holdoutMeanScore: number | null;
|
|
726
|
+
overfitGap: number | null;
|
|
727
|
+
meanCostUsd: number | null;
|
|
728
|
+
p95WallMs: number | null;
|
|
717
729
|
failedRows: number;
|
|
718
730
|
failuresWithAsi: number;
|
|
719
731
|
singleShotTraces: number;
|
|
720
732
|
multiShotTraces: number;
|
|
721
733
|
splitCounts: Record<DatasetSplit, number>;
|
|
722
734
|
domainCounts: Record<string, number>;
|
|
723
|
-
|
|
735
|
+
failureClassCounts: Partial<Record<FailureClass, number>>;
|
|
724
736
|
responsibleSurfaceCounts: Record<string, number>;
|
|
725
737
|
}
|
|
726
738
|
interface ReleaseConfidenceScorecard {
|
|
@@ -994,7 +1006,7 @@ interface FailureClusterReport {
|
|
|
994
1006
|
*/
|
|
995
1007
|
|
|
996
1008
|
interface SummaryTableOptions {
|
|
997
|
-
/** Comparator candidate id. Wilcoxon + Cohen's
|
|
1009
|
+
/** Comparator candidate id. Wilcoxon + paired Cohen's dz are computed
|
|
998
1010
|
* versus this candidate. Required for paired stats columns. */
|
|
999
1011
|
comparator?: string;
|
|
1000
1012
|
/** Which split to read scores from. Default 'holdout'. */
|
|
@@ -1010,10 +1022,16 @@ interface SummaryTableRow {
|
|
|
1010
1022
|
mean: number;
|
|
1011
1023
|
ciLow: number;
|
|
1012
1024
|
ciHigh: number;
|
|
1013
|
-
/** BH-adjusted q-value vs comparator
|
|
1014
|
-
qValue: number;
|
|
1015
|
-
/** Cohen's
|
|
1016
|
-
cohensD: number;
|
|
1025
|
+
/** BH-adjusted q-value vs comparator, or null when unavailable. */
|
|
1026
|
+
qValue: number | null;
|
|
1027
|
+
/** Paired Cohen's dz vs comparator, or null when the paired variance is zero. */
|
|
1028
|
+
cohensD: number | null;
|
|
1029
|
+
/** Matched observations used for paired comparison, or null on the comparator row. */
|
|
1030
|
+
pairedN: number | null;
|
|
1031
|
+
/** Candidate observations without a comparator match. */
|
|
1032
|
+
unpairedCandidateN: number | null;
|
|
1033
|
+
/** Comparator observations without a candidate match. */
|
|
1034
|
+
unpairedComparatorN: number | null;
|
|
1017
1035
|
}
|
|
1018
1036
|
interface SummaryTable {
|
|
1019
1037
|
rows: SummaryTableRow[];
|
|
@@ -1025,7 +1043,7 @@ interface SummaryTable {
|
|
|
1025
1043
|
/**
|
|
1026
1044
|
* Table 1 helper. Buckets runs by `candidateId`, computes mean +
|
|
1027
1045
|
* bootstrap CI on the chosen split, and (when a comparator is given)
|
|
1028
|
-
* BH-adjusted Wilcoxon p + Cohen's
|
|
1046
|
+
* BH-adjusted Wilcoxon p + paired Cohen's dz versus that comparator.
|
|
1029
1047
|
*/
|
|
1030
1048
|
declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
|
|
1031
1049
|
interface ParetoPoint {
|
|
@@ -1041,7 +1059,7 @@ interface ParetoPoint {
|
|
|
1041
1059
|
onFrontier: boolean;
|
|
1042
1060
|
/** Optional gate verdict for this candidate, if a `GateDecision`
|
|
1043
1061
|
* for it was passed in. */
|
|
1044
|
-
gate?: 'promote' | '
|
|
1062
|
+
gate?: 'promote' | 'reject';
|
|
1045
1063
|
}
|
|
1046
1064
|
interface ParetoFigureSpec {
|
|
1047
1065
|
kind: 'pareto-cost-quality';
|
|
@@ -1076,12 +1094,16 @@ interface GainDistributionFigureSpec {
|
|
|
1076
1094
|
split: 'search' | 'holdout';
|
|
1077
1095
|
/** Number of pairs used. */
|
|
1078
1096
|
n: number;
|
|
1097
|
+
/** Candidate rows without a comparator match. */
|
|
1098
|
+
unpairedCandidateN: number;
|
|
1099
|
+
/** Comparator rows without a candidate match. */
|
|
1100
|
+
unpairedComparatorN: number;
|
|
1079
1101
|
bins: GainDistributionBin[];
|
|
1080
|
-
median: number;
|
|
1102
|
+
median: number | null;
|
|
1081
1103
|
ci: {
|
|
1082
1104
|
low: number;
|
|
1083
1105
|
high: number;
|
|
1084
|
-
};
|
|
1106
|
+
} | null;
|
|
1085
1107
|
}
|
|
1086
1108
|
interface GainDistributionOptions {
|
|
1087
1109
|
/** Number of histogram bins. Default 11 (so the centre is exact at 0). */
|
|
@@ -1178,8 +1200,8 @@ interface ResearchReportCandidate {
|
|
|
1178
1200
|
mean: number;
|
|
1179
1201
|
ciLow: number;
|
|
1180
1202
|
ciHigh: number;
|
|
1181
|
-
qValue: number;
|
|
1182
|
-
cohensD: number;
|
|
1203
|
+
qValue: number | null;
|
|
1204
|
+
cohensD: number | null;
|
|
1183
1205
|
meanDeltaVsComparator: number | null;
|
|
1184
1206
|
pairedN: number;
|
|
1185
1207
|
medianGain: number | null;
|
|
@@ -1189,10 +1211,9 @@ interface ResearchReportCandidate {
|
|
|
1189
1211
|
high: number;
|
|
1190
1212
|
} | null;
|
|
1191
1213
|
/**
|
|
1192
|
-
* Bayesian-bootstrap
|
|
1193
|
-
*
|
|
1194
|
-
*
|
|
1195
|
-
* matched pairs."
|
|
1214
|
+
* Bayesian-bootstrap posterior summaries on the paired mean delta.
|
|
1215
|
+
* Dirichlet(1, ..., 1) weights represent uncertainty over the empirical
|
|
1216
|
+
* distribution of matched deltas.
|
|
1196
1217
|
*/
|
|
1197
1218
|
prGreaterThanZero: number | null;
|
|
1198
1219
|
prInRope: number | null;
|
|
@@ -1260,7 +1281,7 @@ interface ResearchReport {
|
|
|
1260
1281
|
* - `summaryTable` marginal stats with BH-FDR-adjusted q-values
|
|
1261
1282
|
* - `paretoChart` cost-vs-quality frontier with gate overlay
|
|
1262
1283
|
* - `gainHistogram` per-candidate paired-delta distribution
|
|
1263
|
-
* - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
|
|
1284
|
+
* - paired posterior (this file): bootstrap CI on median, Bayesian-bootstrap Pr(Δ>0),
|
|
1264
1285
|
* Pr(Δ∈ROPE), MDE at the configured power
|
|
1265
1286
|
*
|
|
1266
1287
|
* Decisions are made on paired evidence — never on marginal means alone —
|
package/dist/reporting.js
CHANGED
|
@@ -4,10 +4,10 @@ import {
|
|
|
4
4
|
evaluateReleaseConfidence,
|
|
5
5
|
judgeReplayGate,
|
|
6
6
|
renderReleaseReport
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-XDWDC2MP.js";
|
|
8
8
|
import {
|
|
9
9
|
rubricPredictiveValidity
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-S5YLIBFX.js";
|
|
11
11
|
import {
|
|
12
12
|
evaluateInterimReleaseConfidence,
|
|
13
13
|
pairedEvalueSequence
|
|
@@ -18,13 +18,15 @@ import {
|
|
|
18
18
|
paretoChart,
|
|
19
19
|
researchReport,
|
|
20
20
|
summaryTable
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-VLOATJQ2.js";
|
|
22
|
+
import "./chunk-DPUHNQLN.js";
|
|
22
23
|
import {
|
|
23
24
|
benjaminiHochberg,
|
|
24
25
|
pairedBootstrap,
|
|
25
26
|
wilcoxonSignedRank
|
|
26
|
-
} from "./chunk-
|
|
27
|
-
import "./chunk-
|
|
27
|
+
} from "./chunk-MHELPNRP.js";
|
|
28
|
+
import "./chunk-2JX3CFMB.js";
|
|
29
|
+
import "./chunk-MA6HLL3S.js";
|
|
28
30
|
import "./chunk-ONWEPEDO.js";
|
|
29
31
|
import "./chunk-PZ5AY32C.js";
|
|
30
32
|
export {
|