@tangle-network/agent-eval 0.128.2 → 0.129.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +265 -0
  2. package/README.md +18 -0
  3. package/dist/analyst/index.d.ts +107 -165
  4. package/dist/analyst/index.js +5 -9
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/belief-state/index.d.ts +2 -19
  7. package/dist/belief-state/index.js +30 -31
  8. package/dist/belief-state/index.js.map +1 -1
  9. package/dist/benchmarks/index.d.ts +5 -8
  10. package/dist/benchmarks/index.js +12 -11
  11. package/dist/builder-eval/index.js +1 -1
  12. package/dist/campaign/index.d.ts +30 -39
  13. package/dist/campaign/index.js +11 -10
  14. package/dist/{chunk-NKAGIDE2.js → chunk-2QU3YOPR.js} +15 -274
  15. package/dist/chunk-2QU3YOPR.js.map +1 -0
  16. package/dist/{chunk-EJGRPCO3.js → chunk-3OCR4R5I.js} +245 -134
  17. package/dist/chunk-3OCR4R5I.js.map +1 -0
  18. package/dist/{chunk-2JX3CFMB.js → chunk-56TAVBOK.js} +5 -2
  19. package/dist/chunk-56TAVBOK.js.map +1 -0
  20. package/dist/{chunk-DPUHNQLN.js → chunk-7FO3TNPI.js} +2 -2
  21. package/dist/{chunk-DJKY2TSY.js → chunk-BSO5JDQH.js} +27 -120
  22. package/dist/chunk-BSO5JDQH.js.map +1 -0
  23. package/dist/{chunk-EZJEIH2R.js → chunk-C6LXANRU.js} +11 -20
  24. package/dist/chunk-C6LXANRU.js.map +1 -0
  25. package/dist/{chunk-ZUUWPZCV.js → chunk-DODXQREJ.js} +4 -4
  26. package/dist/{chunk-2MKQIFS4.js → chunk-E7QXT7SX.js} +2 -2
  27. package/dist/{chunk-NYLOYM6N.js → chunk-EG66UGL4.js} +37 -28
  28. package/dist/chunk-EG66UGL4.js.map +1 -0
  29. package/dist/{chunk-P5W7RQKK.js → chunk-FXTVJPYD.js} +2 -2
  30. package/dist/{chunk-VZSRQ272.js → chunk-G7MGMCZD.js} +6 -2
  31. package/dist/chunk-G7MGMCZD.js.map +1 -0
  32. package/dist/{chunk-IHQDPH7D.js → chunk-H23X7XKK.js} +85 -75
  33. package/dist/chunk-H23X7XKK.js.map +1 -0
  34. package/dist/{chunk-VBQ3CRKH.js → chunk-HPWUNB47.js} +4 -6
  35. package/dist/chunk-HPWUNB47.js.map +1 -0
  36. package/dist/{chunk-XPRT64IE.js → chunk-IYCLP2N2.js} +3 -3
  37. package/dist/chunk-IYCLP2N2.js.map +1 -0
  38. package/dist/{chunk-XDWDC2MP.js → chunk-JQSF5DQT.js} +11 -5
  39. package/dist/chunk-JQSF5DQT.js.map +1 -0
  40. package/dist/{chunk-NACAGYSY.js → chunk-M4YBQKIJ.js} +11 -11
  41. package/dist/chunk-M4YBQKIJ.js.map +1 -0
  42. package/dist/{chunk-YJBNWCAA.js → chunk-NY44NC4A.js} +3 -3
  43. package/dist/chunk-OIUOT4QD.js +44 -0
  44. package/dist/chunk-OIUOT4QD.js.map +1 -0
  45. package/dist/chunk-OWN5NPMC.js +152 -0
  46. package/dist/chunk-OWN5NPMC.js.map +1 -0
  47. package/dist/chunk-PC5DOSM7.js +579 -0
  48. package/dist/chunk-PC5DOSM7.js.map +1 -0
  49. package/dist/{chunk-UB2LOJ6Q.js → chunk-QB6BDBP2.js} +23 -20
  50. package/dist/chunk-QB6BDBP2.js.map +1 -0
  51. package/dist/chunk-RXHCETDZ.js +536 -0
  52. package/dist/chunk-RXHCETDZ.js.map +1 -0
  53. package/dist/{chunk-PBE2LOSS.js → chunk-SFLLL76A.js} +7 -7
  54. package/dist/chunk-SFLLL76A.js.map +1 -0
  55. package/dist/{chunk-VGRCHJON.js → chunk-T6RLYGAD.js} +3 -8
  56. package/dist/chunk-T6RLYGAD.js.map +1 -0
  57. package/dist/{chunk-VLOATJQ2.js → chunk-TJVT4QFF.js} +21 -18
  58. package/dist/chunk-TJVT4QFF.js.map +1 -0
  59. package/dist/{chunk-S5YLIBFX.js → chunk-TQ7LNKZ3.js} +2 -2
  60. package/dist/{chunk-EOSZT7PL.js → chunk-U4L7JRPZ.js} +2 -297
  61. package/dist/chunk-U4L7JRPZ.js.map +1 -0
  62. package/dist/chunk-U4PHLT2N.js +419 -0
  63. package/dist/chunk-U4PHLT2N.js.map +1 -0
  64. package/dist/{chunk-WS3NZZQQ.js → chunk-VCZ5FQYW.js} +3 -4
  65. package/dist/chunk-VCZ5FQYW.js.map +1 -0
  66. package/dist/{chunk-BYT7ELPS.js → chunk-WVATSFCP.js} +2 -2
  67. package/dist/{chunk-TSN7JT6D.js → chunk-X4YIBDER.js} +21 -5
  68. package/dist/{chunk-TSN7JT6D.js.map → chunk-X4YIBDER.js.map} +1 -1
  69. package/dist/{chunk-TBL77AUT.js → chunk-YQN4ICPP.js} +5 -5
  70. package/dist/{chunk-MHELPNRP.js → chunk-ZHTZ4EYI.js} +1 -1
  71. package/dist/chunk-ZHTZ4EYI.js.map +1 -0
  72. package/dist/cli.js +6 -5
  73. package/dist/cli.js.map +1 -1
  74. package/dist/contract/index.d.ts +47 -87
  75. package/dist/contract/index.js +14 -13
  76. package/dist/contract/index.js.map +1 -1
  77. package/dist/control.js +3 -2
  78. package/dist/fuzz.js +3 -2
  79. package/dist/fuzz.js.map +1 -1
  80. package/dist/index.d.ts +659 -203
  81. package/dist/index.js +145 -117
  82. package/dist/index.js.map +1 -1
  83. package/dist/meta-eval/index.js +2 -2
  84. package/dist/multishot/index.d.ts +3 -4
  85. package/dist/multishot/index.js.map +1 -1
  86. package/dist/openapi.json +1 -1
  87. package/dist/pipelines/index.js +5 -5
  88. package/dist/reporting.d.ts +14 -0
  89. package/dist/reporting.js +7 -6
  90. package/dist/rl.d.ts +652 -82
  91. package/dist/rl.js +415 -171
  92. package/dist/rl.js.map +1 -1
  93. package/dist/rollout/index.d.ts +1071 -32
  94. package/dist/rollout/index.js +68 -10
  95. package/dist/run-campaign-OJJ7CZF4.js +18 -0
  96. package/dist/supervisor-run/index.d.ts +114 -4
  97. package/dist/supervisor-run/index.js +4 -3
  98. package/dist/traces.d.ts +1 -1
  99. package/dist/traces.js +6 -5
  100. package/dist/wire/index.d.ts +10 -11
  101. package/dist/wire/index.js +3 -3
  102. package/docs/feature-guide.md +1 -1
  103. package/docs/rollout.md +116 -2
  104. package/package.json +4 -4
  105. package/dist/chunk-2JX3CFMB.js.map +0 -1
  106. package/dist/chunk-DJKY2TSY.js.map +0 -1
  107. package/dist/chunk-EJGRPCO3.js.map +0 -1
  108. package/dist/chunk-EOSZT7PL.js.map +0 -1
  109. package/dist/chunk-EZJEIH2R.js.map +0 -1
  110. package/dist/chunk-IHQDPH7D.js.map +0 -1
  111. package/dist/chunk-MHELPNRP.js.map +0 -1
  112. package/dist/chunk-NACAGYSY.js.map +0 -1
  113. package/dist/chunk-NKAGIDE2.js.map +0 -1
  114. package/dist/chunk-NYLOYM6N.js.map +0 -1
  115. package/dist/chunk-PBE2LOSS.js.map +0 -1
  116. package/dist/chunk-TT4KNT67.js +0 -124
  117. package/dist/chunk-TT4KNT67.js.map +0 -1
  118. package/dist/chunk-UB2LOJ6Q.js.map +0 -1
  119. package/dist/chunk-UWZZKKU7.js +0 -237
  120. package/dist/chunk-UWZZKKU7.js.map +0 -1
  121. package/dist/chunk-VBQ3CRKH.js.map +0 -1
  122. package/dist/chunk-VGRCHJON.js.map +0 -1
  123. package/dist/chunk-VLOATJQ2.js.map +0 -1
  124. package/dist/chunk-VZSRQ272.js.map +0 -1
  125. package/dist/chunk-WS3NZZQQ.js.map +0 -1
  126. package/dist/chunk-XDWDC2MP.js.map +0 -1
  127. package/dist/chunk-XPRT64IE.js.map +0 -1
  128. package/dist/run-campaign-ISHFZ7FJ.js +0 -17
  129. /package/dist/{chunk-DPUHNQLN.js.map → chunk-7FO3TNPI.js.map} +0 -0
  130. /package/dist/{chunk-ZUUWPZCV.js.map → chunk-DODXQREJ.js.map} +0 -0
  131. /package/dist/{chunk-2MKQIFS4.js.map → chunk-E7QXT7SX.js.map} +0 -0
  132. /package/dist/{chunk-P5W7RQKK.js.map → chunk-FXTVJPYD.js.map} +0 -0
  133. /package/dist/{chunk-YJBNWCAA.js.map → chunk-NY44NC4A.js.map} +0 -0
  134. /package/dist/{chunk-S5YLIBFX.js.map → chunk-TQ7LNKZ3.js.map} +0 -0
  135. /package/dist/{chunk-BYT7ELPS.js.map → chunk-WVATSFCP.js.map} +0 -0
  136. /package/dist/{chunk-TBL77AUT.js.map → chunk-YQN4ICPP.js.map} +0 -0
  137. /package/dist/{run-campaign-ISHFZ7FJ.js.map → run-campaign-OJJ7CZF4.js.map} +0 -0
@@ -29,7 +29,7 @@ import {
29
29
  canonicalize,
30
30
  hashJson,
31
31
  validateRunRecord
32
- } from "./chunk-2JX3CFMB.js";
32
+ } from "./chunk-56TAVBOK.js";
33
33
  import {
34
34
  ReplayError
35
35
  } from "./chunk-ONWEPEDO.js";
@@ -787,299 +787,6 @@ function modelHasSnapshot(model) {
787
787
  return false;
788
788
  }
789
789
 
790
- // src/trace/store.ts
791
- var InMemoryTraceStore = class {
792
- runs = /* @__PURE__ */ new Map();
793
- allSpans = [];
794
- allEvents = [];
795
- allArtifacts = [];
796
- allBudget = [];
797
- async appendRun(run) {
798
- if (this.runs.has(run.runId)) throw new Error(`run ${run.runId} already exists`);
799
- this.runs.set(run.runId, { ...run });
800
- }
801
- async updateRun(runId, patch) {
802
- const existing = this.runs.get(runId);
803
- if (!existing) throw new Error(`run ${runId} not found`);
804
- this.runs.set(runId, { ...existing, ...patch });
805
- }
806
- async appendSpan(span) {
807
- this.allSpans.push({ ...span });
808
- }
809
- async updateSpan(spanId, patch) {
810
- const idx = this.allSpans.findIndex((s) => s.spanId === spanId);
811
- if (idx < 0) throw new Error(`span ${spanId} not found`);
812
- this.allSpans[idx] = { ...this.allSpans[idx], ...patch };
813
- }
814
- async appendEvent(event) {
815
- this.allEvents.push({ ...event });
816
- }
817
- async appendArtifact(artifact) {
818
- this.allArtifacts.push({ ...artifact });
819
- }
820
- async appendBudgetEntry(entry) {
821
- this.allBudget.push({ ...entry });
822
- }
823
- async getRun(runId) {
824
- const r = this.runs.get(runId);
825
- return r ? { ...r } : void 0;
826
- }
827
- async listRuns(filter = {}) {
828
- return [...this.runs.values()].filter((r) => matchesRun(r, filter));
829
- }
830
- async spans(filter = {}) {
831
- return this.allSpans.filter((s) => matchesSpan(s, filter)).map((s) => ({ ...s }));
832
- }
833
- async events(filter = {}) {
834
- return this.allEvents.filter((e) => matchesEvent(e, filter)).map((e) => ({ ...e }));
835
- }
836
- async budget(runId) {
837
- return this.allBudget.filter((b) => b.runId === runId).map((b) => ({ ...b }));
838
- }
839
- async artifacts(runId) {
840
- return this.allArtifacts.filter((a) => a.runId === runId).map((a) => ({ ...a }));
841
- }
842
- };
843
- function matchesRun(r, f) {
844
- if (f.scenarioId && r.scenarioId !== f.scenarioId) return false;
845
- if (f.variantId && r.variantId !== f.variantId) return false;
846
- if (f.status && r.status !== f.status) return false;
847
- if (f.since !== void 0 && r.startedAt < f.since) return false;
848
- if (f.until !== void 0 && r.startedAt > f.until) return false;
849
- if (f.tag && r.tags?.[f.tag.key] !== f.tag.value) return false;
850
- if (f.parentRunId && r.parentRunId !== f.parentRunId) return false;
851
- if (f.projectId && r.projectId !== f.projectId) return false;
852
- if (f.chatId && r.chatId !== f.chatId) return false;
853
- if (f.layer && r.layer !== f.layer) return false;
854
- return true;
855
- }
856
- function matchesSpan(s, f) {
857
- if (f.runId && s.runId !== f.runId) return false;
858
- if (f.parentSpanId && s.parentSpanId !== f.parentSpanId) return false;
859
- if (f.kind && s.kind !== f.kind) return false;
860
- if (f.name && s.name !== f.name) return false;
861
- if (f.toolName && (s.kind !== "tool" || s.toolName !== f.toolName)) return false;
862
- if (f.judgeId && (s.kind !== "judge" || s.judgeId !== f.judgeId)) return false;
863
- if (f.since !== void 0 && s.startedAt < f.since) return false;
864
- if (f.until !== void 0 && s.startedAt > f.until) return false;
865
- return true;
866
- }
867
- function matchesEvent(e, f) {
868
- if (f.runId && e.runId !== f.runId) return false;
869
- if (f.spanId && e.spanId !== f.spanId) return false;
870
- if (f.kind && e.kind !== f.kind) return false;
871
- if (f.since !== void 0 && e.timestamp < f.since) return false;
872
- if (f.until !== void 0 && e.timestamp > f.until) return false;
873
- return true;
874
- }
875
- var FileSystemTraceStore = class {
876
- dir;
877
- maxBytes;
878
- /** Lazy in-memory index for queries — populated on first read. */
879
- index;
880
- /** Memoized index build — concurrent first reads share one build, and an
881
- * append racing an in-flight load awaits this so its row isn't lost. */
882
- indexPromise;
883
- /** Strictly-increasing rollover stamp. Date.now() alone collides when two
884
- * rollovers land in the same millisecond, overwriting a rolled file. */
885
- lastRolloverStamp = 0;
886
- /**
887
- * Per-file append serialization. stat → conditional-rename → appendFile is a
888
- * read-modify-write on the active file; without a lock two concurrent appends
889
- * to the same `name` can both pass the size check (exceeding maxBytes) or one
890
- * can append to a file the other just renamed away. Each entry chains the
891
- * next append behind the prior one for that file.
892
- */
893
- appendLocks = /* @__PURE__ */ new Map();
894
- constructor(options) {
895
- this.dir = options.dir;
896
- this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024;
897
- }
898
- async ensureDir() {
899
- const fs = await import("fs/promises");
900
- await fs.mkdir(this.dir, { recursive: true });
901
- }
902
- async append(name, record) {
903
- if (this.indexPromise) await this.indexPromise;
904
- const prior = this.appendLocks.get(name) ?? Promise.resolve();
905
- const result = prior.then(() => this.appendLocked(name, record));
906
- const tail = result.then(
907
- () => {
908
- },
909
- () => {
910
- }
911
- );
912
- this.appendLocks.set(name, tail);
913
- try {
914
- await result;
915
- } finally {
916
- if (this.appendLocks.get(name) === tail) this.appendLocks.delete(name);
917
- }
918
- }
919
- async appendLocked(name, record) {
920
- await this.ensureDir();
921
- const fs = await import("fs/promises");
922
- const path = await import("path");
923
- const active = path.join(this.dir, `${name}.ndjson`);
924
- try {
925
- const stat = await fs.stat(active);
926
- if (stat.size >= this.maxBytes) {
927
- const stamp = Math.max(Date.now(), this.lastRolloverStamp + 1);
928
- this.lastRolloverStamp = stamp;
929
- const rolled = path.join(this.dir, `${name}.${stamp}.ndjson`);
930
- await fs.rename(active, rolled);
931
- }
932
- } catch {
933
- }
934
- await fs.appendFile(active, `${JSON.stringify(record)}
935
- `, "utf8");
936
- if (this.index && !record?._update) {
937
- await this.insertInto(name, record);
938
- }
939
- }
940
- async insertInto(name, record) {
941
- if (!this.index) return;
942
- switch (name) {
943
- case "runs":
944
- await this.index.appendRun(record);
945
- break;
946
- case "spans":
947
- await this.index.appendSpan(record);
948
- break;
949
- case "events":
950
- await this.index.appendEvent(record);
951
- break;
952
- case "artifacts":
953
- await this.index.appendArtifact(record);
954
- break;
955
- case "budget":
956
- await this.index.appendBudgetEntry(record);
957
- break;
958
- }
959
- }
960
- load() {
961
- this.indexPromise ??= this.buildIndex();
962
- return this.indexPromise;
963
- }
964
- async buildIndex() {
965
- const fs = await import("fs/promises");
966
- const path = await import("path");
967
- const store = new InMemoryTraceStore();
968
- let entries;
969
- try {
970
- entries = await fs.readdir(this.dir);
971
- } catch {
972
- this.index = store;
973
- return store;
974
- }
975
- entries.sort((a, b) => {
976
- const baseA = a.split(".")[0];
977
- const baseB = b.split(".")[0];
978
- if (baseA !== baseB) return baseA < baseB ? -1 : 1;
979
- const ta = a.match(/^[^.]+\.(\d+)\.ndjson$/);
980
- const tb = b.match(/^[^.]+\.(\d+)\.ndjson$/);
981
- const tsA = ta ? Number(ta[1]) : Number.POSITIVE_INFINITY;
982
- const tsB = tb ? Number(tb[1]) : Number.POSITIVE_INFINITY;
983
- return tsA - tsB;
984
- });
985
- {
986
- const runUpdates = [];
987
- const spanUpdates = [];
988
- for (const file of entries) {
989
- if (!file.endsWith(".ndjson")) continue;
990
- const full = path.join(this.dir, file);
991
- const content = await fs.readFile(full, "utf8");
992
- const base = file.split(".")[0];
993
- const lines = content.split("\n");
994
- for (let ln = 0; ln < lines.length; ln++) {
995
- const line = lines[ln];
996
- if (!line.trim()) continue;
997
- let record;
998
- try {
999
- record = JSON.parse(line);
1000
- } catch (err) {
1001
- throw new Error(
1002
- `FileSystemTraceStore: corrupt NDJSON in ${file} line ${ln + 1}: ${err instanceof Error ? err.message : String(err)} \u2014 ${line.slice(0, 120)}`
1003
- );
1004
- }
1005
- if (base === "runs") {
1006
- if (record?._update) {
1007
- runUpdates.push(record);
1008
- continue;
1009
- }
1010
- try {
1011
- await store.appendRun(record);
1012
- } catch {
1013
- await store.updateRun(record.runId, record);
1014
- }
1015
- } else if (base === "spans") {
1016
- if (record?._update) {
1017
- spanUpdates.push(record);
1018
- continue;
1019
- }
1020
- await store.appendSpan(record);
1021
- } else if (base === "events") {
1022
- await store.appendEvent(record);
1023
- } else if (base === "artifacts") {
1024
- await store.appendArtifact(record);
1025
- } else if (base === "budget") {
1026
- await store.appendBudgetEntry(record);
1027
- }
1028
- }
1029
- }
1030
- for (const record of runUpdates) {
1031
- await store.updateRun(record.runId, record);
1032
- }
1033
- for (const record of spanUpdates) {
1034
- await store.updateSpan(record.spanId, record);
1035
- }
1036
- }
1037
- this.index = store;
1038
- return store;
1039
- }
1040
- async appendRun(run) {
1041
- await this.append("runs", run);
1042
- }
1043
- async updateRun(runId, patch) {
1044
- await this.append("runs", { runId, ...patch, _update: true });
1045
- if (this.index) await this.index.updateRun(runId, patch);
1046
- }
1047
- async appendSpan(span) {
1048
- await this.append("spans", span);
1049
- }
1050
- async updateSpan(spanId, patch) {
1051
- await this.append("spans", { spanId, ...patch, _update: true });
1052
- if (this.index) await this.index.updateSpan(spanId, patch);
1053
- }
1054
- async appendEvent(event) {
1055
- await this.append("events", event);
1056
- }
1057
- async appendArtifact(artifact) {
1058
- await this.append("artifacts", artifact);
1059
- }
1060
- async appendBudgetEntry(entry) {
1061
- await this.append("budget", entry);
1062
- }
1063
- async getRun(runId) {
1064
- return (await this.load()).getRun(runId);
1065
- }
1066
- async listRuns(filter) {
1067
- return (await this.load()).listRuns(filter);
1068
- }
1069
- async spans(filter) {
1070
- return (await this.load()).spans(filter);
1071
- }
1072
- async events(filter) {
1073
- return (await this.load()).events(filter);
1074
- }
1075
- async budget(runId) {
1076
- return (await this.load()).budget(runId);
1077
- }
1078
- async artifacts(runId) {
1079
- return (await this.load()).artifacts(runId);
1080
- }
1081
- };
1082
-
1083
790
  // src/trace/capture-fetch.ts
1084
791
  var DEFAULT_BODY_CAP = 2 * 1024 * 1024;
1085
792
  function headersToRecord(headers) {
@@ -1984,8 +1691,6 @@ export {
1984
1691
  otlpRowsToRunRecords,
1985
1692
  otlpToTraceRunRecords,
1986
1693
  otlpRowsToTraceRunRecords,
1987
- InMemoryTraceStore,
1988
- FileSystemTraceStore,
1989
1694
  captureFetchToRawSink,
1990
1695
  OTEL_AGENT_EVAL_SCOPE,
1991
1696
  exportRunAsOtlp,
@@ -1998,4 +1703,4 @@ export {
1998
1703
  createReplayFetch,
1999
1704
  iterateRawCalls
2000
1705
  };
2001
- //# sourceMappingURL=chunk-EOSZT7PL.js.map
1706
+ //# sourceMappingURL=chunk-U4L7JRPZ.js.map