@tangle-network/agent-eval 0.191.0 → 0.193.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/CHANGELOG.md +61 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/analyst/index.d.ts +6 -6
  4. package/dist/analyst/index.js +3 -3
  5. package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
  6. package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
  7. package/dist/{benchmark-command-3tZJnD82.js → benchmark-command-DVhUPU7H.js} +7 -7
  8. package/dist/{benchmark-command-3tZJnD82.js.map → benchmark-command-DVhUPU7H.js.map} +1 -1
  9. package/dist/benchmarks/index.d.ts +3 -3
  10. package/dist/benchmarks/index.js +3 -3
  11. package/dist/campaign/index.d.ts +5 -5
  12. package/dist/campaign/index.js +12 -12
  13. package/dist/{campaign-C8iWEF0M.js → campaign-DdE6S80e.js} +15 -328
  14. package/dist/campaign-DdE6S80e.js.map +1 -0
  15. package/dist/{campaign-evidence-BqjlTM3P.js → campaign-evidence-DJc60F9C.js} +11 -46
  16. package/dist/campaign-evidence-DJc60F9C.js.map +1 -0
  17. package/dist/cli.js +47 -10
  18. package/dist/cli.js.map +1 -1
  19. package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
  20. package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
  21. package/dist/contract/index.d.ts +7 -7
  22. package/dist/contract/index.js +10 -10
  23. package/dist/{default-registry-DK_9aVJf.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
  24. package/dist/{default-registry-DK_9aVJf.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
  25. package/dist/{define-agent-eval-P28y7mI1.d.ts → define-agent-eval-DQJvVa1P.d.ts} +15 -24
  26. package/dist/{define-agent-eval-P28y7mI1.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
  27. package/dist/{define-agent-eval-BE43YTm4.js → define-agent-eval-DkPbnJEY.js} +9 -10
  28. package/dist/define-agent-eval-DkPbnJEY.js.map +1 -0
  29. package/dist/diagnosis/index.js +1 -1
  30. package/dist/{dspy-rlm-engine-MfpwoEaj.js → dspy-rlm-engine-BX34vPVY.js} +2 -2
  31. package/dist/{dspy-rlm-engine-MfpwoEaj.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
  32. package/dist/{engine-vNiM02Ub.d.ts → engine-DjRV55F9.d.ts} +2 -2
  33. package/dist/{engine-vNiM02Ub.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
  34. package/dist/evaluation.d.ts +1 -1
  35. package/dist/experiment/index.d.ts +2 -2
  36. package/dist/experiment/index.js +4 -4
  37. package/dist/{external-optimizer-process-xFTliuiK.js → external-optimizer-process-D-Sab4v3.js} +3 -62
  38. package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
  39. package/dist/{external-optimizer-subprocess-B234V9V0.js → external-optimizer-subprocess-kbqC-iWf.js} +64 -3
  40. package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
  41. package/dist/hosted/index.d.ts +2 -2
  42. package/dist/hosted/index.js +1 -1
  43. package/dist/{index-D-TN7ZRM.d.ts → index-0jEtpSM7.d.ts} +3 -3
  44. package/dist/{index-D-TN7ZRM.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
  45. package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
  46. package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
  47. package/dist/{index-CzfnASj_.d.ts → index-DMPLG226.d.ts} +400 -374
  48. package/dist/index-DMPLG226.d.ts.map +1 -0
  49. package/dist/{index-DH-MWH5f.d.ts → index-DWQNDqPd.d.ts} +7 -7
  50. package/dist/{index-DH-MWH5f.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
  51. package/dist/index.d.ts +9 -9
  52. package/dist/index.js +9 -292
  53. package/dist/index.js.map +1 -1
  54. package/dist/jev.d.ts +1 -1
  55. package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
  56. package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
  57. package/dist/ledger-core/index.d.ts +2 -2
  58. package/dist/ledger-core/index.js +2 -2
  59. package/dist/{llm-judge-NJOHM0iA.js → llm-judge-ChbWfeWl.js} +2653 -656
  60. package/dist/llm-judge-ChbWfeWl.js.map +1 -0
  61. package/dist/{matrix-CgqdFm5N.d.ts → matrix-CNu13gg8.d.ts} +2 -2
  62. package/dist/{matrix-CgqdFm5N.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
  63. package/dist/multishot/golden/index.d.ts +1 -1
  64. package/dist/multishot/index.d.ts +2 -2
  65. package/dist/openapi.json +1 -1
  66. package/dist/paired-delta-test-BF-FEP8V.js +78 -0
  67. package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
  68. package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
  69. package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
  70. package/dist/{produced-state-CY-L7CqS.js → produced-state-BqeIaIp2.js} +3 -3
  71. package/dist/{produced-state-CY-L7CqS.js.map → produced-state-BqeIaIp2.js.map} +1 -1
  72. package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
  73. package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
  74. package/dist/rl.d.ts +1 -1
  75. package/dist/rl.js +1 -1
  76. package/dist/{search-ledger-CQ8Uy4FK.js → search-ledger-CJiAEU3j.js} +164 -36
  77. package/dist/search-ledger-CJiAEU3j.js.map +1 -0
  78. package/dist/{search-ledger-file-EvdDZRhv.js → search-ledger-file-DY0AXX3W.js} +2 -2
  79. package/dist/{search-ledger-file-EvdDZRhv.js.map → search-ledger-file-DY0AXX3W.js.map} +1 -1
  80. package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
  81. package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
  82. package/dist/{search-shipper-C55V7HE3.js → search-shipper-DUlwlCsK.js} +3 -3
  83. package/dist/{search-shipper-C55V7HE3.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
  84. package/dist/search-summary-BhY8p0Av.js +378 -0
  85. package/dist/search-summary-BhY8p0Av.js.map +1 -0
  86. package/dist/{semantic-concept-judge-Bt7bXAAo.js → semantic-concept-judge-Bg-560H0.js} +3 -3
  87. package/dist/{semantic-concept-judge-Bt7bXAAo.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
  88. package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
  89. package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
  90. package/dist/{skillopt-optimization-method-DwO3TOAV.js → skillopt-optimization-method-DXQdaBjx.js} +7 -7
  91. package/dist/{skillopt-optimization-method-DwO3TOAV.js.map → skillopt-optimization-method-DXQdaBjx.js.map} +1 -1
  92. package/dist/{statistical-heldout-BED3wBtX.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
  93. package/dist/{statistical-heldout-BED3wBtX.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
  94. package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
  95. package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
  96. package/dist/{store-tool-spans-Cfh9XUCm.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
  97. package/dist/{store-tool-spans-Cfh9XUCm.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
  98. package/dist/{tool-groups-D-mkeN0N.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
  99. package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
  100. package/dist/trace-repair/index.d.ts +1 -1
  101. package/dist/traces.d.ts +2 -2
  102. package/dist/{types-BmQRs661.d.ts → types-BWwRXofp.d.ts} +340 -17
  103. package/dist/types-BWwRXofp.d.ts.map +1 -0
  104. package/dist/{types-ClK-sHs8.d.ts → types-qoxUfKQg.d.ts} +2 -2
  105. package/dist/{types-ClK-sHs8.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
  106. package/docs/campaign-proposers.md +14 -16
  107. package/docs/search-ledger.md +129 -3
  108. package/package.json +1 -1
  109. package/dist/campaign-C8iWEF0M.js.map +0 -1
  110. package/dist/campaign-evidence-BqjlTM3P.js.map +0 -1
  111. package/dist/claim-BGOUqoMq.d.ts +0 -38
  112. package/dist/claim-BGOUqoMq.d.ts.map +0 -1
  113. package/dist/define-agent-eval-BE43YTm4.js.map +0 -1
  114. package/dist/external-optimizer-process-xFTliuiK.js.map +0 -1
  115. package/dist/external-optimizer-subprocess-B234V9V0.js.map +0 -1
  116. package/dist/index-CzfnASj_.d.ts.map +0 -1
  117. package/dist/llm-judge-NJOHM0iA.js.map +0 -1
  118. package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
  119. package/dist/search-ledger-CQ8Uy4FK.js.map +0 -1
  120. package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
  121. package/dist/tool-groups-D-mkeN0N.d.ts.map +0 -1
  122. package/dist/types-BmQRs661.d.ts.map +0 -1
package/dist/jev.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { b as CustomTokenPricing, g as CostReceiptInput } from "./cost-ledger-DbQdN3nO.js";
2
2
  import { i as AnalystFinding, n as AnalystContext, t as Analyst } from "./types-CeDfCRXp.js";
3
- import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-BmQRs661.js";
3
+ import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-BWwRXofp.js";
4
4
  import { EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult, Evaluator, EvaluatorOptions, asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
5
5
  import { JevAnswer, JevAnswerFor, JevJson, JevQuestion, JevQuestions, JevRequest, JevResponseError, JevResult, JevScoreCriteria, JevState, jevUsage, parseJevRequest, parseJevResult } from "./jev-protocol.js";
6
6
  import { z } from "zod";
@@ -1138,6 +1138,98 @@ var FileLedgerJournal = class {
1138
1138
  return first;
1139
1139
  }
1140
1140
  };
1141
+ /**
1142
+ * A journal for runs without a filesystem. It keeps its rows as text in a
1143
+ * store and applies the file journal's rules: the same row parse, hash chain,
1144
+ * idempotent append by `eventId`, and projector transition before a row is
1145
+ * kept. It has no lock, fsync or trusted head, because its rows live and die
1146
+ * with the process that holds the store.
1147
+ */
1148
+ var MemoryLedgerJournal = class {
1149
+ path;
1150
+ codec;
1151
+ store;
1152
+ verified = null;
1153
+ constructor(path, codec, store) {
1154
+ this.path = path;
1155
+ this.codec = codec;
1156
+ this.store = store;
1157
+ }
1158
+ async replay() {
1159
+ return this.guarded(() => this.sync().journal.projector.snapshot());
1160
+ }
1161
+ async append(event) {
1162
+ return this.guarded(() => {
1163
+ const verified = this.sync();
1164
+ const { journal, entries } = verified;
1165
+ const existingSequence = journal.sequenceByEventId.get(event.eventId);
1166
+ if (existingSequence !== void 0) {
1167
+ const existing = entries[existingSequence];
1168
+ if (canonicalString(existing.event) !== canonicalString(event)) throw this.codec.conflictError(`eventId ${event.eventId} already exists with different content`);
1169
+ return {
1170
+ entry: existing,
1171
+ appended: false,
1172
+ projection: journal.projector.snapshot()
1173
+ };
1174
+ }
1175
+ const sequence = entries.length;
1176
+ const material = {
1177
+ ...this.codec.header,
1178
+ sequence,
1179
+ previousHash: journal.head?.entry.entryHash ?? null,
1180
+ event
1181
+ };
1182
+ const text = canonicalString({
1183
+ ...material,
1184
+ entryHash: hashCanonical(material)
1185
+ });
1186
+ const entry = parseRow(text, {
1187
+ path: this.path,
1188
+ line: sequence + 1
1189
+ }, this.codec);
1190
+ admitEntry(journal, entry, Buffer.from(`${text}\n`, "utf8"), this.codec);
1191
+ entries.push(entry);
1192
+ verified.text = `${verified.text}${text}\n`;
1193
+ this.store.write(this.path, verified.text);
1194
+ return {
1195
+ entry,
1196
+ appended: true,
1197
+ projection: journal.projector.snapshot()
1198
+ };
1199
+ });
1200
+ }
1201
+ /** Verify the stored text again whenever it is not the text this instance wrote. */
1202
+ sync() {
1203
+ const text = this.store.read(this.path) ?? "";
1204
+ if (this.verified?.text === text) return this.verified;
1205
+ const journal = emptyJournal(this.codec);
1206
+ const entries = [];
1207
+ const bytes = Buffer.from(text, "utf8");
1208
+ for (const row of splitRows(bytes, this.path, this.codec)) {
1209
+ const entry = parseRow(row.text, {
1210
+ path: this.path,
1211
+ line: entries.length + 1
1212
+ }, this.codec);
1213
+ admitEntry(journal, entry, Buffer.from(bytes.subarray(row.start, row.end)), this.codec);
1214
+ entries.push(entry);
1215
+ }
1216
+ this.verified = {
1217
+ text,
1218
+ journal,
1219
+ entries
1220
+ };
1221
+ return this.verified;
1222
+ }
1223
+ /** A refused transition can leave the projector half applied; discard it. */
1224
+ async guarded(run) {
1225
+ try {
1226
+ return run();
1227
+ } catch (error) {
1228
+ this.verified = null;
1229
+ throw error;
1230
+ }
1231
+ }
1232
+ };
1141
1233
  /** Verify and replay resolved immutable journal bytes without filesystem I/O. */
1142
1234
  function replayLedgerText(text, path, codec) {
1143
1235
  const state = emptyJournal(codec);
@@ -1246,6 +1338,6 @@ function readRange(fd, start, end) {
1246
1338
  return bytes;
1247
1339
  }
1248
1340
  //#endregion
1249
- export { tryAcquireAtomicFileLock as _, verifyEntriesAgainstTrustedHead as a, appendLedgerLine as c, writeLedgerFileAtomically as d, Mutex as f, probeAtomicFileLock as g, AtomicFileLockError as h, trustedHeadPathFor as i, tryWithLedgerFileLock as l, mapConcurrentRange as m, replayLedgerText as n, projectorSnapshotPathFor as o, mapConcurrent as p, readTrustedHeadFile as r, readProjectorSnapshotFile as s, FileLedgerJournal as t, withLedgerFileLock as u };
1341
+ export { probeAtomicFileLock as _, trustedHeadPathFor as a, readProjectorSnapshotFile as c, withLedgerFileLock as d, writeLedgerFileAtomically as f, AtomicFileLockError as g, mapConcurrentRange as h, readTrustedHeadFile as i, appendLedgerLine as l, mapConcurrent as m, MemoryLedgerJournal as n, verifyEntriesAgainstTrustedHead as o, Mutex as p, replayLedgerText as r, projectorSnapshotPathFor as s, FileLedgerJournal as t, tryWithLedgerFileLock as u, tryAcquireAtomicFileLock as v };
1250
1342
 
1251
- //# sourceMappingURL=journal-c3qY0SDP.js.map
1343
+ //# sourceMappingURL=journal-DvKCvjty.js.map