@tangle-network/agent-eval 0.126.7 → 0.128.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +6 -2
- package/dist/analyst/index.d.ts +52 -43
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +35 -40
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -11
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +441 -160
- package/dist/campaign/index.js +11 -10
- package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
- package/dist/chunk-2JX3CFMB.js.map +1 -0
- package/dist/chunk-2MKQIFS4.js +183 -0
- package/dist/chunk-2MKQIFS4.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
- package/dist/chunk-7ZZMD7UK.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
- package/dist/chunk-BYT7ELPS.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/chunk-DRYIUNWY.js +622 -0
- package/dist/chunk-DRYIUNWY.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
- package/dist/chunk-EOSZT7PL.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
- package/dist/chunk-EZJEIH2R.js.map +1 -0
- package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
- package/dist/chunk-IHQDPH7D.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
- package/dist/chunk-NACAGYSY.js.map +1 -0
- package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
- package/dist/chunk-NKAGIDE2.js.map +1 -0
- package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
- package/dist/chunk-NYLOYM6N.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
- package/dist/chunk-TBL77AUT.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
- package/dist/chunk-UB2LOJ6Q.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
- package/dist/chunk-VLOATJQ2.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
- package/dist/chunk-XDWDC2MP.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +438 -138
- package/dist/contract/index.js +259 -113
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +56 -39
- package/dist/control.js +2 -3
- package/dist/hosted/index.d.ts +152 -51
- package/dist/hosted/index.js +24 -1
- package/dist/index.d.ts +399 -277
- package/dist/index.js +363 -244
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +33 -36
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +101 -80
- package/dist/reporting.js +7 -5
- package/dist/rl.d.ts +198 -142
- package/dist/rl.js +235 -127
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +54 -67
- package/dist/rollout/index.js +6 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +72 -50
- package/dist/traces.js +9 -6
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/customer-journeys.md +7 -6
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +78 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-VSMTAMNK.js +0 -53
- package/dist/chunk-VSMTAMNK.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
package/dist/control.d.ts
CHANGED
|
@@ -411,6 +411,8 @@ type ControlDecision<TAction> = {
|
|
|
411
411
|
reason: string;
|
|
412
412
|
pass?: boolean;
|
|
413
413
|
score?: number;
|
|
414
|
+
/** Canonical task-failure class when this stop represents a failed task. */
|
|
415
|
+
failureClass?: FailureClass;
|
|
414
416
|
};
|
|
415
417
|
interface StopDecision {
|
|
416
418
|
stop: boolean;
|
|
@@ -770,7 +772,8 @@ interface AgentProfileCell {
|
|
|
770
772
|
* researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
|
|
771
773
|
* fields are exactly those the paper "Two Loops, Three Roles" requires
|
|
772
774
|
* for reproducibility: who/what/when/cost/seed/hash, plus the search vs
|
|
773
|
-
* holdout split tag
|
|
775
|
+
* holdout split tag. A task score is optional because execution-only records
|
|
776
|
+
* must preserve missing labels instead of converting errors into zero quality.
|
|
774
777
|
*
|
|
775
778
|
* This is intentionally NOT a replacement for the rich `Run` /
|
|
776
779
|
* `ProposeReviewReport` / `ScenarioResult` types already in the
|
|
@@ -791,6 +794,13 @@ interface AgentProfileCell {
|
|
|
791
794
|
/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
|
|
792
795
|
* combined train+test pool that the optimizer is allowed to read. */
|
|
793
796
|
type RunSplitTag = 'search' | 'dev' | 'holdout';
|
|
797
|
+
/**
|
|
798
|
+
* Explicit execution-lifecycle result for a run.
|
|
799
|
+
*
|
|
800
|
+
* This is separate from task quality (`outcome`) and failure classification.
|
|
801
|
+
* Producers set it only from root-run or process evidence.
|
|
802
|
+
*/
|
|
803
|
+
type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
|
|
794
804
|
interface RunTokenUsage {
|
|
795
805
|
input: number;
|
|
796
806
|
/** All generated tokens charged as output, including reasoning tokens. */
|
|
@@ -804,11 +814,6 @@ interface RunTokenUsage {
|
|
|
804
814
|
}
|
|
805
815
|
/**
|
|
806
816
|
* How a run's USD amount was obtained.
|
|
807
|
-
*
|
|
808
|
-
* `costUsd` remains mandatory for wire compatibility. New producers should
|
|
809
|
-
* always populate this discriminated union so a missing bill is never
|
|
810
|
-
* mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
|
|
811
|
-
* the legacy `0` sentinel while this field carries the truthful null.
|
|
812
817
|
*/
|
|
813
818
|
type RunCostProvenance = {
|
|
814
819
|
kind: 'observed';
|
|
@@ -854,8 +859,8 @@ interface JudgeScoresRecord {
|
|
|
854
859
|
perJudge: Record<string, Record<string, number>>;
|
|
855
860
|
/** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
|
|
856
861
|
perDimMean: Record<string, number>;
|
|
857
|
-
/** Composite mean across
|
|
858
|
-
*
|
|
862
|
+
/** Composite mean across successful judges. Mirrors the task score only
|
|
863
|
+
* when `failedJudges` is empty. */
|
|
859
864
|
composite: number;
|
|
860
865
|
/** Judges that errored or returned an unparseable verdict. Recorded
|
|
861
866
|
* by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
|
|
@@ -866,11 +871,11 @@ interface JudgeScoresRecord {
|
|
|
866
871
|
notes?: string;
|
|
867
872
|
}
|
|
868
873
|
interface RunOutcome {
|
|
869
|
-
/** Score on the search/optimization split. Optional
|
|
870
|
-
*
|
|
874
|
+
/** Score on the search/optimization split. Optional for holdout-only and
|
|
875
|
+
* execution-only records. */
|
|
871
876
|
searchScore?: number;
|
|
872
|
-
/** Score on the held-out split. Optional
|
|
873
|
-
*
|
|
877
|
+
/** Score on the held-out split. Optional for search-only and execution-only
|
|
878
|
+
* records. When both scores are absent, the run is explicitly unlabeled. */
|
|
874
879
|
holdoutScore?: number;
|
|
875
880
|
/** Bag of any other metric the run produced — judge dimensions,
|
|
876
881
|
* pass/fail counters, latency stats, etc. Numeric only — keeps
|
|
@@ -934,42 +939,36 @@ interface RunRecord {
|
|
|
934
939
|
wallMs: number;
|
|
935
940
|
/** Time spent queued before execution started, if known. */
|
|
936
941
|
queueMs?: number;
|
|
937
|
-
/** Total USD cost
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
costUsd: number;
|
|
942
|
-
/** Observed, model-priced estimate, or genuinely uncaptured USD amount.
|
|
943
|
-
* Optional only so existing serialized RunRecords remain valid. */
|
|
944
|
-
costProvenance?: RunCostProvenance;
|
|
942
|
+
/** Total USD cost, or null when the producer could not capture one. */
|
|
943
|
+
costUsd: number | null;
|
|
944
|
+
/** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
|
|
945
|
+
costProvenance: RunCostProvenance;
|
|
945
946
|
/** Token usage breakdown. */
|
|
946
947
|
tokenUsage: RunTokenUsage;
|
|
948
|
+
/** Root-run or process terminal result. Never inferred from a child span. */
|
|
949
|
+
terminalOutcome: RunTerminalOutcome;
|
|
950
|
+
/** Root-run or process failure reason. Valid only for a failed, cancelled,
|
|
951
|
+
* or incomplete terminal result; never populated from a child span. */
|
|
952
|
+
terminalFailureReason?: string;
|
|
947
953
|
/** Judge-side metadata, if a judge was used. */
|
|
948
954
|
judgeMetadata?: RunJudgeMetadata;
|
|
949
955
|
/** Per-split scores + raw bag. */
|
|
950
956
|
outcome: RunOutcome;
|
|
951
|
-
/** Canonical
|
|
952
|
-
* `FAILURE_CLASSES` taxonomy.
|
|
953
|
-
*
|
|
954
|
-
*
|
|
955
|
-
* set it via the substrate classifier; leave unset only when the failure
|
|
956
|
-
* genuinely can't be classified. */
|
|
957
|
+
/** Canonical task-failure class drawn from the shared
|
|
958
|
+
* `FAILURE_CLASSES` taxonomy. Producers set it only from task-result
|
|
959
|
+
* evidence. Execution errors belong in
|
|
960
|
+
* `outcome.raw.execution_error_count`. */
|
|
957
961
|
failureClass?: FailureClass;
|
|
958
|
-
/** Free-form
|
|
959
|
-
*
|
|
960
|
-
* The within-agent drill-down; `failureClass` is the cross-agent key. */
|
|
962
|
+
/** Free-form task-failure detail scoped under a non-success
|
|
963
|
+
* `failureClass`. It is invalid without that class. */
|
|
961
964
|
failureMode?: string;
|
|
962
965
|
/** Which split this run was drawn from. */
|
|
963
966
|
splitTag: RunSplitTag;
|
|
964
967
|
/**
|
|
965
|
-
* Stable scenario identifier the run was scored against.
|
|
966
|
-
*
|
|
967
|
-
* that pairs runs by scenario (preferences, paired stats, BT tournament)
|
|
968
|
-
* keys on this. The campaign artifact populates it canonically; legacy
|
|
969
|
-
* runs without it fall back to inference from `outcome.raw.scenario_id`
|
|
970
|
-
* or `experimentId`.
|
|
968
|
+
* Stable scenario identifier the run observed or was scored against.
|
|
969
|
+
* Comparison primitives match this identity rather than input order.
|
|
971
970
|
*/
|
|
972
|
-
scenarioId
|
|
971
|
+
scenarioId: string;
|
|
973
972
|
/**
|
|
974
973
|
* Canonical identity for the agent profile cell that produced this row:
|
|
975
974
|
* profile artifact hash plus optional harness/model/prompt/reporting
|
|
@@ -979,9 +978,27 @@ interface RunRecord {
|
|
|
979
978
|
*/
|
|
980
979
|
agentProfile?: AgentProfileCell;
|
|
981
980
|
}
|
|
981
|
+
/**
|
|
982
|
+
* Canonical task-result classification.
|
|
983
|
+
*
|
|
984
|
+
* A producer may omit classification, record explicit success, or attach
|
|
985
|
+
* domain-specific detail to a non-success class. Detail can never stand alone.
|
|
986
|
+
* Execution errors belong in `outcome.raw.execution_error_count`.
|
|
987
|
+
*/
|
|
988
|
+
type RunTaskFailure = {
|
|
989
|
+
failureClass?: undefined;
|
|
990
|
+
failureMode?: undefined;
|
|
991
|
+
} | {
|
|
992
|
+
failureClass: 'success';
|
|
993
|
+
failureMode?: undefined;
|
|
994
|
+
} | {
|
|
995
|
+
failureClass: Exclude<FailureClass, 'success'>;
|
|
996
|
+
failureMode?: string;
|
|
997
|
+
};
|
|
982
998
|
|
|
983
999
|
interface RunEvidenceMetadata {
|
|
984
1000
|
experimentId: string;
|
|
1001
|
+
scenarioId: string;
|
|
985
1002
|
candidateId: string;
|
|
986
1003
|
seed: number;
|
|
987
1004
|
model: string;
|
|
@@ -990,15 +1007,15 @@ interface RunEvidenceMetadata {
|
|
|
990
1007
|
commitSha: string;
|
|
991
1008
|
splitTag: RunSplitTag;
|
|
992
1009
|
tokenUsage: RunTokenUsage;
|
|
1010
|
+
costProvenance: RunRecord['costProvenance'];
|
|
993
1011
|
queueMs?: number;
|
|
994
1012
|
judgeMetadata?: RunRecord['judgeMetadata'];
|
|
995
1013
|
raw?: Record<string, number>;
|
|
996
1014
|
}
|
|
997
|
-
|
|
1015
|
+
type ControlRunToRunRecordOptions = RunEvidenceMetadata & RunTaskFailure & {
|
|
998
1016
|
runId?: string;
|
|
999
1017
|
score?: number;
|
|
1000
|
-
|
|
1001
|
-
}
|
|
1018
|
+
};
|
|
1002
1019
|
/**
|
|
1003
1020
|
* Project a completed control-loop run into the strict RunRecord shape used by
|
|
1004
1021
|
* release gates, optimizer tables, and research reports.
|
package/dist/control.js
CHANGED
|
@@ -10,11 +10,10 @@ import {
|
|
|
10
10
|
stopOnNoProgress,
|
|
11
11
|
stopOnRepeatedAction,
|
|
12
12
|
subjectiveEval
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-BYT7ELPS.js";
|
|
14
14
|
import "./chunk-VQMK5FMP.js";
|
|
15
|
-
import "./chunk-
|
|
15
|
+
import "./chunk-2JX3CFMB.js";
|
|
16
16
|
import "./chunk-MA6HLL3S.js";
|
|
17
|
-
import "./chunk-VSMTAMNK.js";
|
|
18
17
|
import "./chunk-ONWEPEDO.js";
|
|
19
18
|
import "./chunk-PZ5AY32C.js";
|
|
20
19
|
export {
|
package/dist/hosted/index.d.ts
CHANGED
|
@@ -1,3 +1,41 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Paper-grade RunRecord schema + runtime validator.
|
|
7
|
+
*
|
|
8
|
+
* Every run that participates in a promotion gate, paper table, or
|
|
9
|
+
* researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
|
|
10
|
+
* fields are exactly those the paper "Two Loops, Three Roles" requires
|
|
11
|
+
* for reproducibility: who/what/when/cost/seed/hash, plus the search vs
|
|
12
|
+
* holdout split tag. A task score is optional because execution-only records
|
|
13
|
+
* must preserve missing labels instead of converting errors into zero quality.
|
|
14
|
+
*
|
|
15
|
+
* This is intentionally NOT a replacement for the rich `Run` /
|
|
16
|
+
* `ProposeReviewReport` / `ScenarioResult` types already in the
|
|
17
|
+
* package. Those are runtime structures with full provenance. A
|
|
18
|
+
* `RunRecord` is the analysis-time projection — the JSON-friendly
|
|
19
|
+
* row you'd put in a parquet file or paste into a notebook.
|
|
20
|
+
*
|
|
21
|
+
* Validate at the boundary:
|
|
22
|
+
*
|
|
23
|
+
* const rec = validateRunRecord(rawJson) // throws on missing
|
|
24
|
+
* const ok = isRunRecord(rawJson) // boolean check
|
|
25
|
+
* const rec = parseRunRecordSafe(rawJson) // { ok, value | error }
|
|
26
|
+
*
|
|
27
|
+
* The validator runs in pure TS — zod is intentionally NOT a
|
|
28
|
+
* dependency. Round-trip tested in `tests/run-record.test.ts`.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Explicit execution-lifecycle result for a run.
|
|
33
|
+
*
|
|
34
|
+
* This is separate from task quality (`outcome`) and failure classification.
|
|
35
|
+
* Producers set it only from root-run or process evidence.
|
|
36
|
+
*/
|
|
37
|
+
type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
|
|
38
|
+
|
|
1
39
|
/**
|
|
2
40
|
* Pass A substrate types — `runCampaign` is the one primitive every
|
|
3
41
|
* eval flow composes from. Three contracts in this file:
|
|
@@ -102,7 +140,7 @@ interface ParetoPoint {
|
|
|
102
140
|
onFrontier: boolean;
|
|
103
141
|
/** Optional gate verdict for this candidate, if a `GateDecision`
|
|
104
142
|
* for it was passed in. */
|
|
105
|
-
gate?: 'promote' | '
|
|
143
|
+
gate?: 'promote' | 'reject';
|
|
106
144
|
}
|
|
107
145
|
interface ParetoFigureSpec {
|
|
108
146
|
kind: 'pareto-cost-quality';
|
|
@@ -176,8 +214,8 @@ interface InsightReport {
|
|
|
176
214
|
/** Number of runs analyzed. */
|
|
177
215
|
n: number;
|
|
178
216
|
/** Runtime facts carried by the run records. These describe execution,
|
|
179
|
-
* not task quality: duration, queueing, token categories, models,
|
|
180
|
-
*
|
|
217
|
+
* not task quality: duration, queueing, token categories, models,
|
|
218
|
+
* execution errors, and terminal outcomes. */
|
|
181
219
|
execution: ExecutionInsight;
|
|
182
220
|
/** Composite-score distribution across all runs. Always present. */
|
|
183
221
|
composite: ScalarDistribution;
|
|
@@ -235,11 +273,10 @@ interface InsightReport {
|
|
|
235
273
|
* Surfaced metrics: composite, cost, duration, tokenUsage, plus any
|
|
236
274
|
* per-dimension judge metric present in both windows. */
|
|
237
275
|
priorPeriodComparison?: PriorPeriodComparison;
|
|
238
|
-
/** Model-free failure
|
|
239
|
-
* by count descending.
|
|
240
|
-
*
|
|
241
|
-
|
|
242
|
-
failureModes?: FailureModeTally[];
|
|
276
|
+
/** Model-free task-failure breakdown from `RunRecord.failureClass`, ranked
|
|
277
|
+
* by count descending. Domain-specific `failureMode` detail is retained on
|
|
278
|
+
* each record but never creates a second aggregation vocabulary. */
|
|
279
|
+
failureClasses?: FailureClassTally[];
|
|
243
280
|
/** Top-N actionable recommendations, ranked by priority. The packet's
|
|
244
281
|
* human-readable layer; the numeric sections are the evidence. */
|
|
245
282
|
recommendations: Recommendation[];
|
|
@@ -287,15 +324,46 @@ interface ExecutionInsight {
|
|
|
287
324
|
events: number;
|
|
288
325
|
reportingRuns: number;
|
|
289
326
|
};
|
|
290
|
-
/**
|
|
291
|
-
*
|
|
292
|
-
|
|
327
|
+
/** Runs with explicit execution-error telemetry. This is independent of
|
|
328
|
+
* whether the root run ultimately succeeded, failed, or has no terminal
|
|
329
|
+
* evidence. */
|
|
330
|
+
executionErrors: {
|
|
293
331
|
runs: number;
|
|
294
|
-
|
|
295
|
-
|
|
332
|
+
/** Share among runs that supplied an execution-error count.
|
|
333
|
+
* `null` when no run supplied error telemetry. */
|
|
334
|
+
fraction: number | null;
|
|
335
|
+
/** Execution-error events reported through the canonical count. */
|
|
336
|
+
events: number;
|
|
337
|
+
/** Runs that supplied an execution-error count, including explicit zeroes. */
|
|
296
338
|
reportingRuns: number;
|
|
339
|
+
/** Exact sum of `outcome.raw.error_span_count`, kept separate from other errors. */
|
|
340
|
+
errorSpanEvents: number;
|
|
341
|
+
/** Runs that supplied `outcome.raw.error_span_count`, including explicit zeroes. */
|
|
342
|
+
errorSpanReportingRuns: number;
|
|
343
|
+
/**
|
|
344
|
+
* Error-telemetry coverage crossed with independently reported terminal
|
|
345
|
+
* outcomes. `unreported` is distinct from a reported zero.
|
|
346
|
+
*/
|
|
347
|
+
byTerminalOutcome: Record<RunTerminalOutcome, ExecutionErrorOutcomeCell>;
|
|
348
|
+
};
|
|
349
|
+
/** Root-run or process outcomes. Missing `RunRecord.terminalOutcome` values
|
|
350
|
+
* count as `unknown`; child-span status never changes these counts. */
|
|
351
|
+
terminalOutcomes: {
|
|
352
|
+
succeeded: number;
|
|
353
|
+
failed: number;
|
|
354
|
+
cancelled: number;
|
|
355
|
+
incomplete: number;
|
|
356
|
+
unknown: number;
|
|
297
357
|
};
|
|
298
358
|
}
|
|
359
|
+
interface ExecutionErrorOutcomeCell {
|
|
360
|
+
/** Runs that explicitly reported one or more execution errors. */
|
|
361
|
+
withErrors: number;
|
|
362
|
+
/** Runs that explicitly reported zero execution errors. */
|
|
363
|
+
withoutErrors: number;
|
|
364
|
+
/** Runs with no execution-error count from the producer. */
|
|
365
|
+
unreported: number;
|
|
366
|
+
}
|
|
299
367
|
interface TokenUsageInsight {
|
|
300
368
|
input: ScalarDistribution;
|
|
301
369
|
output: ScalarDistribution;
|
|
@@ -314,12 +382,18 @@ interface TokenUsageInsight {
|
|
|
314
382
|
interface ScalarDistribution {
|
|
315
383
|
/** Sample count after dropping non-finite values. */
|
|
316
384
|
n: number;
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
385
|
+
/** Null when `n` is zero. */
|
|
386
|
+
mean: number | null;
|
|
387
|
+
/** Null when `n` is zero. */
|
|
388
|
+
p50: number | null;
|
|
389
|
+
/** Null when `n` is zero. */
|
|
390
|
+
p95: number | null;
|
|
391
|
+
/** Null when `n` is zero. */
|
|
392
|
+
stddev: number | null;
|
|
393
|
+
/** Null when `n` is zero. */
|
|
394
|
+
min: number | null;
|
|
395
|
+
/** Null when `n` is zero. */
|
|
396
|
+
max: number | null;
|
|
323
397
|
/** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
|
|
324
398
|
histogram: GainDistributionBin[];
|
|
325
399
|
/** Worst-N runs by score, ascending. Populated for the composite
|
|
@@ -385,12 +459,16 @@ interface LiftInsight {
|
|
|
385
459
|
pValue: number;
|
|
386
460
|
/** Number of paired observations. */
|
|
387
461
|
n: number;
|
|
388
|
-
/**
|
|
389
|
-
|
|
462
|
+
/** Scored baseline observations without a candidate match. */
|
|
463
|
+
unpairedBaseline: number;
|
|
464
|
+
/** Scored candidate observations without a baseline match. */
|
|
465
|
+
unpairedCandidate: number;
|
|
466
|
+
/** Cohen's dz for paired deltas; null when the observed delta variance is zero. */
|
|
467
|
+
cohensD: number | null;
|
|
390
468
|
/** Minimum detectable effect at current n, 80% power. */
|
|
391
469
|
mde: number;
|
|
392
|
-
/**
|
|
393
|
-
requiredN: number;
|
|
470
|
+
/** Paired sample size needed to detect the standardized effect at 80% power. */
|
|
471
|
+
requiredN: number | null;
|
|
394
472
|
}
|
|
395
473
|
interface FailureClusterInsight {
|
|
396
474
|
/** All clusters identified by the registry, ranked by share descending. */
|
|
@@ -406,15 +484,13 @@ interface FailureClusterInsight {
|
|
|
406
484
|
}>;
|
|
407
485
|
totalFailures: number;
|
|
408
486
|
}
|
|
409
|
-
/** Model-free failure breakdown over
|
|
410
|
-
*
|
|
411
|
-
*
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
/**
|
|
416
|
-
mode: string;
|
|
417
|
-
/** Number of runs carrying this tag. */
|
|
487
|
+
/** Model-free task-failure breakdown over canonical `RunRecord.failureClass`
|
|
488
|
+
* values. Unlike semantic failure clusters, this is computed directly from
|
|
489
|
+
* run records and does not require a model analyst. */
|
|
490
|
+
interface FailureClassTally {
|
|
491
|
+
/** Canonical task-failure class. */
|
|
492
|
+
failureClass: FailureClass;
|
|
493
|
+
/** Number of failed runs carrying this class. */
|
|
418
494
|
count: number;
|
|
419
495
|
/** Share of the whole corpus, 0..1. */
|
|
420
496
|
share: number;
|
|
@@ -454,7 +530,7 @@ interface ReleaseSummary {
|
|
|
454
530
|
status: 'pass' | 'warn' | 'fail';
|
|
455
531
|
axes: Array<{
|
|
456
532
|
name: 'quality-lift' | 'contamination' | 'composite-distribution';
|
|
457
|
-
status: 'pass' | 'warn' | 'fail';
|
|
533
|
+
status: 'pass' | 'warn' | 'fail' | 'not_evaluated';
|
|
458
534
|
detail: string;
|
|
459
535
|
}>;
|
|
460
536
|
/** Free-form issues surfaced beyond the standard axes. Empty by default;
|
|
@@ -514,9 +590,8 @@ interface Recommendation {
|
|
|
514
590
|
* # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
|
|
515
591
|
* a partner's self-hosted one, a future open implementation) must accept.
|
|
516
592
|
*
|
|
517
|
-
*
|
|
518
|
-
*
|
|
519
|
-
* (`HostedWireVersion` literal increment).
|
|
593
|
+
* This package implements exactly one wire version. Servers reject every
|
|
594
|
+
* other version instead of translating old payloads.
|
|
520
595
|
*
|
|
521
596
|
* The wire format is two event streams in one transport:
|
|
522
597
|
*
|
|
@@ -535,7 +610,7 @@ interface Recommendation {
|
|
|
535
610
|
* ever sees another tenant's data.
|
|
536
611
|
*/
|
|
537
612
|
|
|
538
|
-
declare const HOSTED_WIRE_VERSION: "2026-
|
|
613
|
+
declare const HOSTED_WIRE_VERSION: "2026-07-24.v1";
|
|
539
614
|
type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
|
|
540
615
|
/** Every ingest request carries these. */
|
|
541
616
|
interface HostedIngestHeaders {
|
|
@@ -545,8 +620,8 @@ interface HostedIngestHeaders {
|
|
|
545
620
|
'x-tangle-tenant-id': string;
|
|
546
621
|
/** Wire-version pin so the server can reject incompatible payloads. */
|
|
547
622
|
'x-tangle-wire-version': HostedWireVersion;
|
|
548
|
-
/**
|
|
549
|
-
'idempotency-key'
|
|
623
|
+
/** Stable request key generated once and reused across retries. */
|
|
624
|
+
'idempotency-key': string;
|
|
550
625
|
}
|
|
551
626
|
/** Lifecycle stages of an eval-run as the substrate reports them. */
|
|
552
627
|
type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
|
|
@@ -555,11 +630,15 @@ interface EvalRunCellScore {
|
|
|
555
630
|
scenarioId: string;
|
|
556
631
|
/** Repetition index when reps > 1; 0 for the default. */
|
|
557
632
|
rep: number;
|
|
558
|
-
/** Composite score across
|
|
559
|
-
compositeMean: number;
|
|
560
|
-
/** Per-judge
|
|
633
|
+
/** Composite score across successful judges, or null when unscored. */
|
|
634
|
+
compositeMean: number | null;
|
|
635
|
+
/** Per-judge and per-dimension scores; failed or missing judges are absent. */
|
|
561
636
|
dimensions: Record<string, Record<string, number>>;
|
|
562
|
-
/**
|
|
637
|
+
/** Root execution result, kept separate from task quality. */
|
|
638
|
+
terminalOutcome: RunTerminalOutcome;
|
|
639
|
+
/** Canonical execution-error count, or null when the producer did not measure it. */
|
|
640
|
+
executionErrorCount: number | null;
|
|
641
|
+
/** Per-cell dispatch or judge error. Missing on success. */
|
|
563
642
|
errorMessage?: string;
|
|
564
643
|
}
|
|
565
644
|
interface EvalRunGenerationSnapshot {
|
|
@@ -573,8 +652,8 @@ interface EvalRunGenerationSnapshot {
|
|
|
573
652
|
surface?: MutableSurface;
|
|
574
653
|
/** Per-cell scores for this generation. */
|
|
575
654
|
cells: EvalRunCellScore[];
|
|
576
|
-
/**
|
|
577
|
-
compositeMean: number;
|
|
655
|
+
/** Mean across scored cells, or null when no cell has a task-quality label. */
|
|
656
|
+
compositeMean: number | null;
|
|
578
657
|
/** Total $ spent across this generation. */
|
|
579
658
|
costUsd: number;
|
|
580
659
|
/** Wall-clock duration of this generation. */
|
|
@@ -614,10 +693,14 @@ interface EvalRunEvent {
|
|
|
614
693
|
* paired-bootstrap lift CI, judge stats, inter-rater agreement,
|
|
615
694
|
* contamination check, failure clusters (when an analyst is wired),
|
|
616
695
|
* outcome correlation (when downstream signal is supplied), and the
|
|
617
|
-
* recommendations the dashboard surfaces verbatim.
|
|
618
|
-
* clients that don't know about this field continue to work. */
|
|
696
|
+
* recommendations the dashboard surfaces verbatim. */
|
|
619
697
|
insightReport?: InsightReport;
|
|
620
698
|
}
|
|
699
|
+
/**
|
|
700
|
+
* Canonical unsigned 64-bit integer encoded as a base-10 string.
|
|
701
|
+
* JSON numbers cannot represent OTLP nanosecond timestamps exactly.
|
|
702
|
+
*/
|
|
703
|
+
type UnixNanoTimestamp = string;
|
|
621
704
|
/**
|
|
622
705
|
* OTel-shape span with a few additional attributes for eval-run pivoting.
|
|
623
706
|
* Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
|
|
@@ -628,11 +711,11 @@ interface TraceSpanEvent {
|
|
|
628
711
|
spanId: string;
|
|
629
712
|
parentSpanId?: string;
|
|
630
713
|
name: string;
|
|
631
|
-
startTimeUnixNano:
|
|
632
|
-
endTimeUnixNano:
|
|
714
|
+
startTimeUnixNano: UnixNanoTimestamp;
|
|
715
|
+
endTimeUnixNano: UnixNanoTimestamp;
|
|
633
716
|
attributes: Record<string, string | number | boolean>;
|
|
634
717
|
events?: Array<{
|
|
635
|
-
timeUnixNano:
|
|
718
|
+
timeUnixNano: UnixNanoTimestamp;
|
|
636
719
|
name: string;
|
|
637
720
|
attributes?: Record<string, string | number | boolean>;
|
|
638
721
|
}>;
|
|
@@ -740,4 +823,22 @@ declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
|
|
|
740
823
|
env?: Record<string, string | undefined>;
|
|
741
824
|
}): HostedClient | undefined;
|
|
742
825
|
|
|
743
|
-
|
|
826
|
+
declare const UnixNanoTimestampSchema: z.ZodType<UnixNanoTimestamp>;
|
|
827
|
+
declare const InsightReportSchema: z.ZodType<InsightReport>;
|
|
828
|
+
declare const MutableSurfaceSchema: z.ZodType<MutableSurface>;
|
|
829
|
+
declare const RunTerminalOutcomeSchema: z.ZodEnum<{
|
|
830
|
+
succeeded: "succeeded";
|
|
831
|
+
unknown: "unknown";
|
|
832
|
+
failed: "failed";
|
|
833
|
+
cancelled: "cancelled";
|
|
834
|
+
incomplete: "incomplete";
|
|
835
|
+
}>;
|
|
836
|
+
declare const EvalRunCellScoreSchema: z.ZodType<EvalRunCellScore>;
|
|
837
|
+
declare const EvalRunGenerationSnapshotSchema: z.ZodType<EvalRunGenerationSnapshot>;
|
|
838
|
+
declare const EvalRunEventSchema: z.ZodType<EvalRunEvent>;
|
|
839
|
+
declare const TraceSpanEventSchema: z.ZodType<TraceSpanEvent>;
|
|
840
|
+
declare const IngestEvalRunsRequestSchema: z.ZodType<IngestEvalRunsRequest>;
|
|
841
|
+
declare const IngestTracesRequestSchema: z.ZodType<IngestTracesRequest>;
|
|
842
|
+
declare const IngestResponseSchema: z.ZodType<IngestResponse>;
|
|
843
|
+
|
|
844
|
+
export { type EvalRunCellScore, EvalRunCellScoreSchema, type EvalRunEvent, EvalRunEventSchema, type EvalRunGenerationSnapshot, EvalRunGenerationSnapshotSchema, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, IngestEvalRunsRequestSchema, type IngestResponse, IngestResponseSchema, type IngestTracesRequest, IngestTracesRequestSchema, type InsightReport, InsightReportSchema, MutableSurfaceSchema, RunTerminalOutcomeSchema, type TraceSpanEvent, TraceSpanEventSchema, type UnixNanoTimestamp, UnixNanoTimestampSchema, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };
|
package/dist/hosted/index.js
CHANGED
|
@@ -1,12 +1,35 @@
|
|
|
1
1
|
import {
|
|
2
|
+
EvalRunCellScoreSchema,
|
|
3
|
+
EvalRunEventSchema,
|
|
4
|
+
EvalRunGenerationSnapshotSchema,
|
|
2
5
|
HOSTED_WIRE_VERSION,
|
|
6
|
+
IngestEvalRunsRequestSchema,
|
|
7
|
+
IngestResponseSchema,
|
|
8
|
+
IngestTracesRequestSchema,
|
|
9
|
+
InsightReportSchema,
|
|
10
|
+
MutableSurfaceSchema,
|
|
11
|
+
RunTerminalOutcomeSchema,
|
|
12
|
+
TraceSpanEventSchema,
|
|
13
|
+
UnixNanoTimestampSchema,
|
|
3
14
|
createHostedClient,
|
|
4
15
|
hostedClientFromEnv,
|
|
5
16
|
hostedTenantFromEnv
|
|
6
|
-
} from "../chunk-
|
|
17
|
+
} from "../chunk-DRYIUNWY.js";
|
|
18
|
+
import "../chunk-MA6HLL3S.js";
|
|
7
19
|
import "../chunk-PZ5AY32C.js";
|
|
8
20
|
export {
|
|
21
|
+
EvalRunCellScoreSchema,
|
|
22
|
+
EvalRunEventSchema,
|
|
23
|
+
EvalRunGenerationSnapshotSchema,
|
|
9
24
|
HOSTED_WIRE_VERSION,
|
|
25
|
+
IngestEvalRunsRequestSchema,
|
|
26
|
+
IngestResponseSchema,
|
|
27
|
+
IngestTracesRequestSchema,
|
|
28
|
+
InsightReportSchema,
|
|
29
|
+
MutableSurfaceSchema,
|
|
30
|
+
RunTerminalOutcomeSchema,
|
|
31
|
+
TraceSpanEventSchema,
|
|
32
|
+
UnixNanoTimestampSchema,
|
|
10
33
|
createHostedClient,
|
|
11
34
|
hostedClientFromEnv,
|
|
12
35
|
hostedTenantFromEnv
|