@tangle-network/agent-eval 0.170.0 → 0.172.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/README.md +3 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +2 -2
- package/dist/{experiment-tracker-Dm8yQMqb.d.ts → attestation-CJBGmMVh.d.ts} +78 -2
- package/dist/attestation-CJBGmMVh.d.ts.map +1 -0
- package/dist/{experiment-tracker-BKEumQug.js → attestation-XSUpbc4o.js} +96 -2
- package/dist/attestation-XSUpbc4o.js.map +1 -0
- package/dist/{benchmark-command-BA7qOdWw.js → benchmark-command-DoFcisuM.js} +2 -2
- package/dist/{benchmark-command-BA7qOdWw.js.map → benchmark-command-DoFcisuM.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +2 -2
- package/dist/bounded-process-CVOC_D3H.js +181 -0
- package/dist/bounded-process-CVOC_D3H.js.map +1 -0
- package/dist/builder-eval/index.js +39 -98
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -7
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-BeCbxFqs.js → campaign-Dp35pBbS.js} +4 -4
- package/dist/{campaign-BeCbxFqs.js.map → campaign-Dp35pBbS.js.map} +1 -1
- package/dist/canonical-CFpojCN5.d.ts +31 -0
- package/dist/canonical-CFpojCN5.d.ts.map +1 -0
- package/dist/{chat-client-DEtybj5i.js → chat-client-DI79OPye.js} +2 -2
- package/dist/{chat-client-DEtybj5i.js.map → chat-client-DI79OPye.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-_Fsa5c2_.d.ts → client-Df7wdslk.d.ts} +2 -2
- package/dist/{client-_Fsa5c2_.d.ts.map → client-Df7wdslk.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +4 -4
- package/dist/{default-registry-ovxrOP0_.d.ts → default-registry-XxedTLwu.d.ts} +3 -3
- package/dist/{default-registry-ovxrOP0_.d.ts.map → default-registry-XxedTLwu.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DVJm8Xlh.d.ts → define-agent-eval-0wW7gFhr.d.ts} +4 -4
- package/dist/{define-agent-eval-DVJm8Xlh.d.ts.map → define-agent-eval-0wW7gFhr.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Clj-8igZ.js → define-agent-eval-jS8xj_Q_.js} +2 -2
- package/dist/{define-agent-eval-Clj-8igZ.js.map → define-agent-eval-jS8xj_Q_.js.map} +1 -1
- package/dist/descriptive-B2iPaT9J.d.ts +89 -0
- package/dist/descriptive-B2iPaT9J.d.ts.map +1 -0
- package/dist/{engine-D12Rb6WB.d.ts → engine-BfRay1qD.d.ts} +2 -2
- package/dist/{engine-D12Rb6WB.d.ts.map → engine-BfRay1qD.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -54
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +1 -95
- package/dist/experiment/index.js.map +1 -1
- package/dist/{heldout-gate-Bn7_xWCv.d.ts → heldout-gate-JgNRDZwZ.d.ts} +3 -3
- package/dist/{heldout-gate-Bn7_xWCv.d.ts.map → heldout-gate-JgNRDZwZ.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/{index-lfaSeKSD.d.ts → index-D-UdhAmg.d.ts} +3 -31
- package/dist/index-D-UdhAmg.d.ts.map +1 -0
- package/dist/{index-CM-SM00y.d.ts → index-DDAPhUJJ.d.ts} +4 -4
- package/dist/{index-CM-SM00y.d.ts.map → index-DDAPhUJJ.d.ts.map} +1 -1
- package/dist/{index-Bfs5aufo.d.ts → index-DnglhM0A.d.ts} +9 -9
- package/dist/{index-Bfs5aufo.d.ts.map → index-DnglhM0A.d.ts.map} +1 -1
- package/dist/{index-DBbivBNs.d.ts → index-_vPrVMRX.d.ts} +6 -6
- package/dist/{index-DBbivBNs.d.ts.map → index-_vPrVMRX.d.ts.map} +1 -1
- package/dist/index.d.ts +245 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +712 -9
- package/dist/index.js.map +1 -1
- package/dist/{integrity-CyWSSoQS.js → integrity-BWywb34E.js} +34 -11
- package/dist/{integrity-CyWSSoQS.js.map → integrity-BWywb34E.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -1
- package/dist/{llm-judge-DbJdo8Nj.js → llm-judge-aQHIk5_-.js} +20 -11
- package/dist/llm-judge-aQHIk5_-.js.map +1 -0
- package/dist/{matrix-BpI5Trmo.d.ts → matrix-Ch8JO1pG.d.ts} +2 -2
- package/dist/{matrix-BpI5Trmo.d.ts.map → matrix-Ch8JO1pG.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +162 -2
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +287 -2
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-CtSIp5cQ.js → produced-state-CxmbFxFd.js} +2 -2
- package/dist/{produced-state-CtSIp5cQ.js.map → produced-state-CxmbFxFd.js.map} +1 -1
- package/dist/{promotion-policy-CkXSgKkF.d.ts → promotion-policy-BBBcz5_3.d.ts} +2 -2
- package/dist/{promotion-policy-CkXSgKkF.d.ts.map → promotion-policy-BBBcz5_3.d.ts.map} +1 -1
- package/dist/{provenance-CIRUardl.d.ts → provenance-Dp-vvyrU.d.ts} +17 -5
- package/dist/provenance-Dp-vvyrU.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{skillopt-optimization-method-B2R9C5aG.js → skillopt-optimization-method-LHi02MzH.js} +2 -2
- package/dist/{skillopt-optimization-method-B2R9C5aG.js.map → skillopt-optimization-method-LHi02MzH.js.map} +1 -1
- package/dist/{statistical-heldout-DFS7QGpS.d.ts → statistical-heldout-Yldkntvy.d.ts} +2 -2
- package/dist/{statistical-heldout-DFS7QGpS.d.ts.map → statistical-heldout-Yldkntvy.d.ts.map} +1 -1
- package/dist/{store-tool-spans-B2DJ_82T.d.ts → store-tool-spans-BvdUbeOB.d.ts} +3 -3
- package/dist/{store-tool-spans-B2DJ_82T.d.ts.map → store-tool-spans-BvdUbeOB.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +35 -2
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +83 -38
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{tool-groups-BnXlCJZQ.d.ts → tool-groups-DjwlMBvW.d.ts} +2 -2
- package/dist/tool-groups-DjwlMBvW.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-i21ccEkr.d.ts → types-CCZ34qmV.d.ts} +2 -2
- package/dist/{types-i21ccEkr.d.ts.map → types-CCZ34qmV.d.ts.map} +1 -1
- package/dist/{types-DeIUdzNd.d.ts → types-CoPUTiXb.d.ts} +23 -92
- package/dist/types-CoPUTiXb.d.ts.map +1 -0
- package/dist/{types-Dy237wiH.d.ts → types-nokrtr7M.d.ts} +21 -1
- package/dist/types-nokrtr7M.d.ts.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.d.ts.map +1 -1
- package/docs/campaign-proposers.md +4 -0
- package/docs/eval-surface-map.md +36 -0
- package/docs/insight-report.md +19 -0
- package/docs/plants.md +123 -0
- package/docs/public-api.md +45 -20
- package/package.json +19 -18
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-Dm8yQMqb.d.ts.map +0 -1
- package/dist/index-lfaSeKSD.d.ts.map +0 -1
- package/dist/llm-judge-DbJdo8Nj.js.map +0 -1
- package/dist/provenance-CIRUardl.d.ts.map +0 -1
- package/dist/tool-groups-BnXlCJZQ.d.ts.map +0 -1
- package/dist/types-DeIUdzNd.d.ts.map +0 -1
- package/dist/types-Dy237wiH.d.ts.map +0 -1
|
@@ -780,9 +780,12 @@ function readSpend(v) {
|
|
|
780
780
|
output: num(tok.output),
|
|
781
781
|
cacheRead: num(cacheRead),
|
|
782
782
|
cacheWrite: num(cacheWrite),
|
|
783
|
-
hasCache: cacheRead !== void 0 || cacheWrite !== void 0
|
|
783
|
+
hasCache: cacheRead !== void 0 || cacheWrite !== void 0,
|
|
784
|
+
breakdownKnown: tok.cacheBreakdownKnown !== false
|
|
784
785
|
},
|
|
785
|
-
|
|
786
|
+
tokensKnown: rec.tokensKnown !== false,
|
|
787
|
+
usd: num(rec.usd),
|
|
788
|
+
usdKnown: rec.usdKnown !== false
|
|
786
789
|
};
|
|
787
790
|
}
|
|
788
791
|
function parseJsonl(text) {
|
|
@@ -927,7 +930,12 @@ function parseSupervisorTree(src) {
|
|
|
927
930
|
let brainCacheRead = 0;
|
|
928
931
|
let brainCacheWrite = 0;
|
|
929
932
|
let brainHasCache = false;
|
|
933
|
+
let brainCacheBreakdownKnown = true;
|
|
934
|
+
let brainTokensKnownCount = 0;
|
|
935
|
+
let brainTokensUnknownCount = 0;
|
|
930
936
|
let brainUsd = 0;
|
|
937
|
+
let brainUsdKnownCount = 0;
|
|
938
|
+
let brainUsdUnknownCount = 0;
|
|
931
939
|
let meteredCount = 0;
|
|
932
940
|
let rootId = null;
|
|
933
941
|
let unreadableRows = 0;
|
|
@@ -1006,9 +1014,12 @@ function parseSupervisorTree(src) {
|
|
|
1006
1014
|
output: 0,
|
|
1007
1015
|
cacheRead: 0,
|
|
1008
1016
|
cacheWrite: 0,
|
|
1009
|
-
hasCache: false
|
|
1017
|
+
hasCache: false,
|
|
1018
|
+
breakdownKnown: true
|
|
1010
1019
|
},
|
|
1011
|
-
|
|
1020
|
+
tokensKnown: true,
|
|
1021
|
+
usd: 0,
|
|
1022
|
+
usdKnown: true
|
|
1012
1023
|
},
|
|
1013
1024
|
hasSpend: false
|
|
1014
1025
|
});
|
|
@@ -1021,12 +1032,19 @@ function parseSupervisorTree(src) {
|
|
|
1021
1032
|
}
|
|
1022
1033
|
for (const row of meteredRows) {
|
|
1023
1034
|
if (rootId === null || row.id !== rootId) continue;
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1035
|
+
if (row.spend.tokensKnown) {
|
|
1036
|
+
brainIn += row.spend.tokens.input;
|
|
1037
|
+
brainOut += row.spend.tokens.output;
|
|
1038
|
+
brainCacheRead += row.spend.tokens.cacheRead;
|
|
1039
|
+
brainCacheWrite += row.spend.tokens.cacheWrite;
|
|
1040
|
+
brainHasCache = brainHasCache || row.spend.tokens.hasCache;
|
|
1041
|
+
brainCacheBreakdownKnown = brainCacheBreakdownKnown && row.spend.tokens.breakdownKnown;
|
|
1042
|
+
brainTokensKnownCount += 1;
|
|
1043
|
+
} else brainTokensUnknownCount += 1;
|
|
1044
|
+
if (row.spend.usdKnown) {
|
|
1045
|
+
brainUsd += row.spend.usd;
|
|
1046
|
+
brainUsdKnownCount += 1;
|
|
1047
|
+
} else brainUsdUnknownCount += 1;
|
|
1030
1048
|
meteredCount += 1;
|
|
1031
1049
|
}
|
|
1032
1050
|
const workerSpawns = spawns.filter((s) => s.valid && s.id !== rootId);
|
|
@@ -1150,7 +1168,12 @@ function parseSupervisorTree(src) {
|
|
|
1150
1168
|
cacheRead: brainCacheRead,
|
|
1151
1169
|
cacheWrite: brainCacheWrite,
|
|
1152
1170
|
hasCache: brainHasCache,
|
|
1171
|
+
cacheBreakdownKnown: brainCacheBreakdownKnown,
|
|
1172
|
+
tokensKnownCount: brainTokensKnownCount,
|
|
1173
|
+
tokensUnknownCount: brainTokensUnknownCount,
|
|
1153
1174
|
usd: brainUsd,
|
|
1175
|
+
usdKnownCount: brainUsdKnownCount,
|
|
1176
|
+
usdUnknownCount: brainUsdUnknownCount,
|
|
1154
1177
|
meteredCount
|
|
1155
1178
|
},
|
|
1156
1179
|
workerLogs,
|
|
@@ -1492,4 +1515,4 @@ function analyzeSupervisorRunIntegrity(input, options = {}) {
|
|
|
1492
1515
|
//#endregion
|
|
1493
1516
|
export { parseJsonl as a, SUPERVISOR_RUN_INTEGRITY_SCHEMA as c, parseJson as i, supervisorRunRolloutLines as n, parseSupervisorTree as o, asRecord as r, workerSourceKey as s, analyzeSupervisorRunIntegrity as t };
|
|
1494
1517
|
|
|
1495
|
-
//# sourceMappingURL=integrity-
|
|
1518
|
+
//# sourceMappingURL=integrity-BWywb34E.js.map
|