@tangle-network/agent-eval 0.170.0 → 0.172.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +68 -0
  2. package/README.md +3 -0
  3. package/dist/adapters/http.d.ts +2 -2
  4. package/dist/analyst/index.d.ts +5 -5
  5. package/dist/analyst/index.js +2 -2
  6. package/dist/{experiment-tracker-Dm8yQMqb.d.ts → attestation-CJBGmMVh.d.ts} +78 -2
  7. package/dist/attestation-CJBGmMVh.d.ts.map +1 -0
  8. package/dist/{experiment-tracker-BKEumQug.js → attestation-XSUpbc4o.js} +96 -2
  9. package/dist/attestation-XSUpbc4o.js.map +1 -0
  10. package/dist/{benchmark-command-BA7qOdWw.js → benchmark-command-DoFcisuM.js} +2 -2
  11. package/dist/{benchmark-command-BA7qOdWw.js.map → benchmark-command-DoFcisuM.js.map} +1 -1
  12. package/dist/benchmarks/index.d.ts +4 -4
  13. package/dist/benchmarks/index.js +2 -2
  14. package/dist/bounded-process-CVOC_D3H.js +181 -0
  15. package/dist/bounded-process-CVOC_D3H.js.map +1 -0
  16. package/dist/builder-eval/index.js +39 -98
  17. package/dist/builder-eval/index.js.map +1 -1
  18. package/dist/campaign/index.d.ts +8 -7
  19. package/dist/campaign/index.js +4 -4
  20. package/dist/{campaign-BeCbxFqs.js → campaign-Dp35pBbS.js} +4 -4
  21. package/dist/{campaign-BeCbxFqs.js.map → campaign-Dp35pBbS.js.map} +1 -1
  22. package/dist/canonical-CFpojCN5.d.ts +31 -0
  23. package/dist/canonical-CFpojCN5.d.ts.map +1 -0
  24. package/dist/{chat-client-DEtybj5i.js → chat-client-DI79OPye.js} +2 -2
  25. package/dist/{chat-client-DEtybj5i.js.map → chat-client-DI79OPye.js.map} +1 -1
  26. package/dist/cli.js +1 -1
  27. package/dist/{client-_Fsa5c2_.d.ts → client-Df7wdslk.d.ts} +2 -2
  28. package/dist/{client-_Fsa5c2_.d.ts.map → client-Df7wdslk.d.ts.map} +1 -1
  29. package/dist/contract/index.d.ts +9 -9
  30. package/dist/contract/index.js +4 -4
  31. package/dist/{default-registry-ovxrOP0_.d.ts → default-registry-XxedTLwu.d.ts} +3 -3
  32. package/dist/{default-registry-ovxrOP0_.d.ts.map → default-registry-XxedTLwu.d.ts.map} +1 -1
  33. package/dist/{define-agent-eval-DVJm8Xlh.d.ts → define-agent-eval-0wW7gFhr.d.ts} +4 -4
  34. package/dist/{define-agent-eval-DVJm8Xlh.d.ts.map → define-agent-eval-0wW7gFhr.d.ts.map} +1 -1
  35. package/dist/{define-agent-eval-Clj-8igZ.js → define-agent-eval-jS8xj_Q_.js} +2 -2
  36. package/dist/{define-agent-eval-Clj-8igZ.js.map → define-agent-eval-jS8xj_Q_.js.map} +1 -1
  37. package/dist/descriptive-B2iPaT9J.d.ts +89 -0
  38. package/dist/descriptive-B2iPaT9J.d.ts.map +1 -0
  39. package/dist/{engine-D12Rb6WB.d.ts → engine-BfRay1qD.d.ts} +2 -2
  40. package/dist/{engine-D12Rb6WB.d.ts.map → engine-BfRay1qD.d.ts.map} +1 -1
  41. package/dist/experiment/index.d.ts +3 -54
  42. package/dist/experiment/index.d.ts.map +1 -1
  43. package/dist/experiment/index.js +1 -95
  44. package/dist/experiment/index.js.map +1 -1
  45. package/dist/{heldout-gate-Bn7_xWCv.d.ts → heldout-gate-JgNRDZwZ.d.ts} +3 -3
  46. package/dist/{heldout-gate-Bn7_xWCv.d.ts.map → heldout-gate-JgNRDZwZ.d.ts.map} +1 -1
  47. package/dist/hosted/index.d.ts +1 -1
  48. package/dist/{index-lfaSeKSD.d.ts → index-D-UdhAmg.d.ts} +3 -31
  49. package/dist/index-D-UdhAmg.d.ts.map +1 -0
  50. package/dist/{index-CM-SM00y.d.ts → index-DDAPhUJJ.d.ts} +4 -4
  51. package/dist/{index-CM-SM00y.d.ts.map → index-DDAPhUJJ.d.ts.map} +1 -1
  52. package/dist/{index-Bfs5aufo.d.ts → index-DnglhM0A.d.ts} +9 -9
  53. package/dist/{index-Bfs5aufo.d.ts.map → index-DnglhM0A.d.ts.map} +1 -1
  54. package/dist/{index-DBbivBNs.d.ts → index-_vPrVMRX.d.ts} +6 -6
  55. package/dist/{index-DBbivBNs.d.ts.map → index-_vPrVMRX.d.ts.map} +1 -1
  56. package/dist/index.d.ts +245 -15
  57. package/dist/index.d.ts.map +1 -1
  58. package/dist/index.js +712 -9
  59. package/dist/index.js.map +1 -1
  60. package/dist/{integrity-CyWSSoQS.js → integrity-BWywb34E.js} +34 -11
  61. package/dist/{integrity-CyWSSoQS.js.map → integrity-BWywb34E.js.map} +1 -1
  62. package/dist/ledger-core/index.d.ts +2 -1
  63. package/dist/{llm-judge-DbJdo8Nj.js → llm-judge-aQHIk5_-.js} +20 -11
  64. package/dist/llm-judge-aQHIk5_-.js.map +1 -0
  65. package/dist/{matrix-BpI5Trmo.d.ts → matrix-Ch8JO1pG.d.ts} +2 -2
  66. package/dist/{matrix-BpI5Trmo.d.ts.map → matrix-Ch8JO1pG.d.ts.map} +1 -1
  67. package/dist/meta-eval/index.d.ts +162 -2
  68. package/dist/meta-eval/index.d.ts.map +1 -1
  69. package/dist/meta-eval/index.js +287 -2
  70. package/dist/meta-eval/index.js.map +1 -1
  71. package/dist/multishot/golden/index.d.ts +1 -1
  72. package/dist/multishot/index.d.ts +2 -2
  73. package/dist/openapi.json +1 -1
  74. package/dist/{produced-state-CtSIp5cQ.js → produced-state-CxmbFxFd.js} +2 -2
  75. package/dist/{produced-state-CtSIp5cQ.js.map → produced-state-CxmbFxFd.js.map} +1 -1
  76. package/dist/{promotion-policy-CkXSgKkF.d.ts → promotion-policy-BBBcz5_3.d.ts} +2 -2
  77. package/dist/{promotion-policy-CkXSgKkF.d.ts.map → promotion-policy-BBBcz5_3.d.ts.map} +1 -1
  78. package/dist/{provenance-CIRUardl.d.ts → provenance-Dp-vvyrU.d.ts} +17 -5
  79. package/dist/provenance-Dp-vvyrU.d.ts.map +1 -0
  80. package/dist/rl.d.ts +1 -1
  81. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
  82. package/dist/{skillopt-optimization-method-B2R9C5aG.js → skillopt-optimization-method-LHi02MzH.js} +2 -2
  83. package/dist/{skillopt-optimization-method-B2R9C5aG.js.map → skillopt-optimization-method-LHi02MzH.js.map} +1 -1
  84. package/dist/{statistical-heldout-DFS7QGpS.d.ts → statistical-heldout-Yldkntvy.d.ts} +2 -2
  85. package/dist/{statistical-heldout-DFS7QGpS.d.ts.map → statistical-heldout-Yldkntvy.d.ts.map} +1 -1
  86. package/dist/{store-tool-spans-B2DJ_82T.d.ts → store-tool-spans-BvdUbeOB.d.ts} +3 -3
  87. package/dist/{store-tool-spans-B2DJ_82T.d.ts.map → store-tool-spans-BvdUbeOB.d.ts.map} +1 -1
  88. package/dist/supervisor-run/index.d.ts +35 -2
  89. package/dist/supervisor-run/index.d.ts.map +1 -1
  90. package/dist/supervisor-run/index.js +83 -38
  91. package/dist/supervisor-run/index.js.map +1 -1
  92. package/dist/{tool-groups-BnXlCJZQ.d.ts → tool-groups-DjwlMBvW.d.ts} +2 -2
  93. package/dist/tool-groups-DjwlMBvW.d.ts.map +1 -0
  94. package/dist/trace-repair/index.d.ts +1 -1
  95. package/dist/traces.d.ts +2 -2
  96. package/dist/{types-i21ccEkr.d.ts → types-CCZ34qmV.d.ts} +2 -2
  97. package/dist/{types-i21ccEkr.d.ts.map → types-CCZ34qmV.d.ts.map} +1 -1
  98. package/dist/{types-DeIUdzNd.d.ts → types-CoPUTiXb.d.ts} +23 -92
  99. package/dist/types-CoPUTiXb.d.ts.map +1 -0
  100. package/dist/{types-Dy237wiH.d.ts → types-nokrtr7M.d.ts} +21 -1
  101. package/dist/types-nokrtr7M.d.ts.map +1 -0
  102. package/dist/wire/index.d.ts +3 -3
  103. package/dist/wire/index.d.ts.map +1 -1
  104. package/docs/campaign-proposers.md +4 -0
  105. package/docs/eval-surface-map.md +36 -0
  106. package/docs/insight-report.md +19 -0
  107. package/docs/plants.md +123 -0
  108. package/docs/public-api.md +45 -20
  109. package/package.json +19 -18
  110. package/dist/experiment-tracker-BKEumQug.js.map +0 -1
  111. package/dist/experiment-tracker-Dm8yQMqb.d.ts.map +0 -1
  112. package/dist/index-lfaSeKSD.d.ts.map +0 -1
  113. package/dist/llm-judge-DbJdo8Nj.js.map +0 -1
  114. package/dist/provenance-CIRUardl.d.ts.map +0 -1
  115. package/dist/tool-groups-BnXlCJZQ.d.ts.map +0 -1
  116. package/dist/types-DeIUdzNd.d.ts.map +0 -1
  117. package/dist/types-Dy237wiH.d.ts.map +0 -1
@@ -780,9 +780,12 @@ function readSpend(v) {
780
780
  output: num(tok.output),
781
781
  cacheRead: num(cacheRead),
782
782
  cacheWrite: num(cacheWrite),
783
- hasCache: cacheRead !== void 0 || cacheWrite !== void 0
783
+ hasCache: cacheRead !== void 0 || cacheWrite !== void 0,
784
+ breakdownKnown: tok.cacheBreakdownKnown !== false
784
785
  },
785
- usd: num(rec.usd)
786
+ tokensKnown: rec.tokensKnown !== false,
787
+ usd: num(rec.usd),
788
+ usdKnown: rec.usdKnown !== false
786
789
  };
787
790
  }
788
791
  function parseJsonl(text) {
@@ -927,7 +930,12 @@ function parseSupervisorTree(src) {
927
930
  let brainCacheRead = 0;
928
931
  let brainCacheWrite = 0;
929
932
  let brainHasCache = false;
933
+ let brainCacheBreakdownKnown = true;
934
+ let brainTokensKnownCount = 0;
935
+ let brainTokensUnknownCount = 0;
930
936
  let brainUsd = 0;
937
+ let brainUsdKnownCount = 0;
938
+ let brainUsdUnknownCount = 0;
931
939
  let meteredCount = 0;
932
940
  let rootId = null;
933
941
  let unreadableRows = 0;
@@ -1006,9 +1014,12 @@ function parseSupervisorTree(src) {
1006
1014
  output: 0,
1007
1015
  cacheRead: 0,
1008
1016
  cacheWrite: 0,
1009
- hasCache: false
1017
+ hasCache: false,
1018
+ breakdownKnown: true
1010
1019
  },
1011
- usd: 0
1020
+ tokensKnown: true,
1021
+ usd: 0,
1022
+ usdKnown: true
1012
1023
  },
1013
1024
  hasSpend: false
1014
1025
  });
@@ -1021,12 +1032,19 @@ function parseSupervisorTree(src) {
1021
1032
  }
1022
1033
  for (const row of meteredRows) {
1023
1034
  if (rootId === null || row.id !== rootId) continue;
1024
- brainIn += row.spend.tokens.input;
1025
- brainOut += row.spend.tokens.output;
1026
- brainCacheRead += row.spend.tokens.cacheRead;
1027
- brainCacheWrite += row.spend.tokens.cacheWrite;
1028
- brainHasCache = brainHasCache || row.spend.tokens.hasCache;
1029
- brainUsd += row.spend.usd;
1035
+ if (row.spend.tokensKnown) {
1036
+ brainIn += row.spend.tokens.input;
1037
+ brainOut += row.spend.tokens.output;
1038
+ brainCacheRead += row.spend.tokens.cacheRead;
1039
+ brainCacheWrite += row.spend.tokens.cacheWrite;
1040
+ brainHasCache = brainHasCache || row.spend.tokens.hasCache;
1041
+ brainCacheBreakdownKnown = brainCacheBreakdownKnown && row.spend.tokens.breakdownKnown;
1042
+ brainTokensKnownCount += 1;
1043
+ } else brainTokensUnknownCount += 1;
1044
+ if (row.spend.usdKnown) {
1045
+ brainUsd += row.spend.usd;
1046
+ brainUsdKnownCount += 1;
1047
+ } else brainUsdUnknownCount += 1;
1030
1048
  meteredCount += 1;
1031
1049
  }
1032
1050
  const workerSpawns = spawns.filter((s) => s.valid && s.id !== rootId);
@@ -1150,7 +1168,12 @@ function parseSupervisorTree(src) {
1150
1168
  cacheRead: brainCacheRead,
1151
1169
  cacheWrite: brainCacheWrite,
1152
1170
  hasCache: brainHasCache,
1171
+ cacheBreakdownKnown: brainCacheBreakdownKnown,
1172
+ tokensKnownCount: brainTokensKnownCount,
1173
+ tokensUnknownCount: brainTokensUnknownCount,
1153
1174
  usd: brainUsd,
1175
+ usdKnownCount: brainUsdKnownCount,
1176
+ usdUnknownCount: brainUsdUnknownCount,
1154
1177
  meteredCount
1155
1178
  },
1156
1179
  workerLogs,
@@ -1492,4 +1515,4 @@ function analyzeSupervisorRunIntegrity(input, options = {}) {
1492
1515
  //#endregion
1493
1516
  export { parseJsonl as a, SUPERVISOR_RUN_INTEGRITY_SCHEMA as c, parseJson as i, supervisorRunRolloutLines as n, parseSupervisorTree as o, asRecord as r, workerSourceKey as s, analyzeSupervisorRunIntegrity as t };
1494
1517
 
1495
- //# sourceMappingURL=integrity-CyWSSoQS.js.map
1518
+ //# sourceMappingURL=integrity-BWywb34E.js.map