@tangle-network/agent-eval 0.191.0 → 0.193.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +6 -6
- package/dist/analyst/index.js +3 -3
- package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
- package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
- package/dist/{benchmark-command-3tZJnD82.js → benchmark-command-DVhUPU7H.js} +7 -7
- package/dist/{benchmark-command-3tZJnD82.js.map → benchmark-command-DVhUPU7H.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +12 -12
- package/dist/{campaign-C8iWEF0M.js → campaign-DdE6S80e.js} +15 -328
- package/dist/campaign-DdE6S80e.js.map +1 -0
- package/dist/{campaign-evidence-BqjlTM3P.js → campaign-evidence-DJc60F9C.js} +11 -46
- package/dist/campaign-evidence-DJc60F9C.js.map +1 -0
- package/dist/cli.js +47 -10
- package/dist/cli.js.map +1 -1
- package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
- package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +10 -10
- package/dist/{default-registry-DK_9aVJf.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
- package/dist/{default-registry-DK_9aVJf.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
- package/dist/{define-agent-eval-P28y7mI1.d.ts → define-agent-eval-DQJvVa1P.d.ts} +15 -24
- package/dist/{define-agent-eval-P28y7mI1.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BE43YTm4.js → define-agent-eval-DkPbnJEY.js} +9 -10
- package/dist/define-agent-eval-DkPbnJEY.js.map +1 -0
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-MfpwoEaj.js → dspy-rlm-engine-BX34vPVY.js} +2 -2
- package/dist/{dspy-rlm-engine-MfpwoEaj.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
- package/dist/{engine-vNiM02Ub.d.ts → engine-DjRV55F9.d.ts} +2 -2
- package/dist/{engine-vNiM02Ub.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +4 -4
- package/dist/{external-optimizer-process-xFTliuiK.js → external-optimizer-process-D-Sab4v3.js} +3 -62
- package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
- package/dist/{external-optimizer-subprocess-B234V9V0.js → external-optimizer-subprocess-kbqC-iWf.js} +64 -3
- package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +1 -1
- package/dist/{index-D-TN7ZRM.d.ts → index-0jEtpSM7.d.ts} +3 -3
- package/dist/{index-D-TN7ZRM.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
- package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
- package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
- package/dist/{index-CzfnASj_.d.ts → index-DMPLG226.d.ts} +400 -374
- package/dist/index-DMPLG226.d.ts.map +1 -0
- package/dist/{index-DH-MWH5f.d.ts → index-DWQNDqPd.d.ts} +7 -7
- package/dist/{index-DH-MWH5f.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +9 -292
- package/dist/index.js.map +1 -1
- package/dist/jev.d.ts +1 -1
- package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
- package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-NJOHM0iA.js → llm-judge-ChbWfeWl.js} +2653 -656
- package/dist/llm-judge-ChbWfeWl.js.map +1 -0
- package/dist/{matrix-CgqdFm5N.d.ts → matrix-CNu13gg8.d.ts} +2 -2
- package/dist/{matrix-CgqdFm5N.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/paired-delta-test-BF-FEP8V.js +78 -0
- package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
- package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
- package/dist/{produced-state-CY-L7CqS.js → produced-state-BqeIaIp2.js} +3 -3
- package/dist/{produced-state-CY-L7CqS.js.map → produced-state-BqeIaIp2.js.map} +1 -1
- package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
- package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +1 -1
- package/dist/{search-ledger-CQ8Uy4FK.js → search-ledger-CJiAEU3j.js} +164 -36
- package/dist/search-ledger-CJiAEU3j.js.map +1 -0
- package/dist/{search-ledger-file-EvdDZRhv.js → search-ledger-file-DY0AXX3W.js} +2 -2
- package/dist/{search-ledger-file-EvdDZRhv.js.map → search-ledger-file-DY0AXX3W.js.map} +1 -1
- package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
- package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
- package/dist/{search-shipper-C55V7HE3.js → search-shipper-DUlwlCsK.js} +3 -3
- package/dist/{search-shipper-C55V7HE3.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
- package/dist/search-summary-BhY8p0Av.js +378 -0
- package/dist/search-summary-BhY8p0Av.js.map +1 -0
- package/dist/{semantic-concept-judge-Bt7bXAAo.js → semantic-concept-judge-Bg-560H0.js} +3 -3
- package/dist/{semantic-concept-judge-Bt7bXAAo.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
- package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
- package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DwO3TOAV.js → skillopt-optimization-method-DXQdaBjx.js} +7 -7
- package/dist/{skillopt-optimization-method-DwO3TOAV.js.map → skillopt-optimization-method-DXQdaBjx.js.map} +1 -1
- package/dist/{statistical-heldout-BED3wBtX.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
- package/dist/{statistical-heldout-BED3wBtX.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
- package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
- package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
- package/dist/{tool-groups-D-mkeN0N.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
- package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-BmQRs661.d.ts → types-BWwRXofp.d.ts} +340 -17
- package/dist/types-BWwRXofp.d.ts.map +1 -0
- package/dist/{types-ClK-sHs8.d.ts → types-qoxUfKQg.d.ts} +2 -2
- package/dist/{types-ClK-sHs8.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
- package/docs/campaign-proposers.md +14 -16
- package/docs/search-ledger.md +129 -3
- package/package.json +1 -1
- package/dist/campaign-C8iWEF0M.js.map +0 -1
- package/dist/campaign-evidence-BqjlTM3P.js.map +0 -1
- package/dist/claim-BGOUqoMq.d.ts +0 -38
- package/dist/claim-BGOUqoMq.d.ts.map +0 -1
- package/dist/define-agent-eval-BE43YTm4.js.map +0 -1
- package/dist/external-optimizer-process-xFTliuiK.js.map +0 -1
- package/dist/external-optimizer-subprocess-B234V9V0.js.map +0 -1
- package/dist/index-CzfnASj_.d.ts.map +0 -1
- package/dist/llm-judge-NJOHM0iA.js.map +0 -1
- package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
- package/dist/search-ledger-CQ8Uy4FK.js.map +0 -1
- package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
- package/dist/tool-groups-D-mkeN0N.d.ts.map +0 -1
- package/dist/types-BmQRs661.d.ts.map +0 -1
package/dist/jev.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { b as CustomTokenPricing, g as CostReceiptInput } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { i as AnalystFinding, n as AnalystContext, t as Analyst } from "./types-CeDfCRXp.js";
|
|
3
|
-
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-
|
|
3
|
+
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-BWwRXofp.js";
|
|
4
4
|
import { EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult, Evaluator, EvaluatorOptions, asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
|
|
5
5
|
import { JevAnswer, JevAnswerFor, JevJson, JevQuestion, JevQuestions, JevRequest, JevResponseError, JevResult, JevScoreCriteria, JevState, jevUsage, parseJevRequest, parseJevResult } from "./jev-protocol.js";
|
|
6
6
|
import { z } from "zod";
|
|
@@ -1138,6 +1138,98 @@ var FileLedgerJournal = class {
|
|
|
1138
1138
|
return first;
|
|
1139
1139
|
}
|
|
1140
1140
|
};
|
|
1141
|
+
/**
|
|
1142
|
+
* A journal for runs without a filesystem. It keeps its rows as text in a
|
|
1143
|
+
* store and applies the file journal's rules: the same row parse, hash chain,
|
|
1144
|
+
* idempotent append by `eventId`, and projector transition before a row is
|
|
1145
|
+
* kept. It has no lock, fsync or trusted head, because its rows live and die
|
|
1146
|
+
* with the process that holds the store.
|
|
1147
|
+
*/
|
|
1148
|
+
var MemoryLedgerJournal = class {
|
|
1149
|
+
path;
|
|
1150
|
+
codec;
|
|
1151
|
+
store;
|
|
1152
|
+
verified = null;
|
|
1153
|
+
constructor(path, codec, store) {
|
|
1154
|
+
this.path = path;
|
|
1155
|
+
this.codec = codec;
|
|
1156
|
+
this.store = store;
|
|
1157
|
+
}
|
|
1158
|
+
async replay() {
|
|
1159
|
+
return this.guarded(() => this.sync().journal.projector.snapshot());
|
|
1160
|
+
}
|
|
1161
|
+
async append(event) {
|
|
1162
|
+
return this.guarded(() => {
|
|
1163
|
+
const verified = this.sync();
|
|
1164
|
+
const { journal, entries } = verified;
|
|
1165
|
+
const existingSequence = journal.sequenceByEventId.get(event.eventId);
|
|
1166
|
+
if (existingSequence !== void 0) {
|
|
1167
|
+
const existing = entries[existingSequence];
|
|
1168
|
+
if (canonicalString(existing.event) !== canonicalString(event)) throw this.codec.conflictError(`eventId ${event.eventId} already exists with different content`);
|
|
1169
|
+
return {
|
|
1170
|
+
entry: existing,
|
|
1171
|
+
appended: false,
|
|
1172
|
+
projection: journal.projector.snapshot()
|
|
1173
|
+
};
|
|
1174
|
+
}
|
|
1175
|
+
const sequence = entries.length;
|
|
1176
|
+
const material = {
|
|
1177
|
+
...this.codec.header,
|
|
1178
|
+
sequence,
|
|
1179
|
+
previousHash: journal.head?.entry.entryHash ?? null,
|
|
1180
|
+
event
|
|
1181
|
+
};
|
|
1182
|
+
const text = canonicalString({
|
|
1183
|
+
...material,
|
|
1184
|
+
entryHash: hashCanonical(material)
|
|
1185
|
+
});
|
|
1186
|
+
const entry = parseRow(text, {
|
|
1187
|
+
path: this.path,
|
|
1188
|
+
line: sequence + 1
|
|
1189
|
+
}, this.codec);
|
|
1190
|
+
admitEntry(journal, entry, Buffer.from(`${text}\n`, "utf8"), this.codec);
|
|
1191
|
+
entries.push(entry);
|
|
1192
|
+
verified.text = `${verified.text}${text}\n`;
|
|
1193
|
+
this.store.write(this.path, verified.text);
|
|
1194
|
+
return {
|
|
1195
|
+
entry,
|
|
1196
|
+
appended: true,
|
|
1197
|
+
projection: journal.projector.snapshot()
|
|
1198
|
+
};
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
/** Verify the stored text again whenever it is not the text this instance wrote. */
|
|
1202
|
+
sync() {
|
|
1203
|
+
const text = this.store.read(this.path) ?? "";
|
|
1204
|
+
if (this.verified?.text === text) return this.verified;
|
|
1205
|
+
const journal = emptyJournal(this.codec);
|
|
1206
|
+
const entries = [];
|
|
1207
|
+
const bytes = Buffer.from(text, "utf8");
|
|
1208
|
+
for (const row of splitRows(bytes, this.path, this.codec)) {
|
|
1209
|
+
const entry = parseRow(row.text, {
|
|
1210
|
+
path: this.path,
|
|
1211
|
+
line: entries.length + 1
|
|
1212
|
+
}, this.codec);
|
|
1213
|
+
admitEntry(journal, entry, Buffer.from(bytes.subarray(row.start, row.end)), this.codec);
|
|
1214
|
+
entries.push(entry);
|
|
1215
|
+
}
|
|
1216
|
+
this.verified = {
|
|
1217
|
+
text,
|
|
1218
|
+
journal,
|
|
1219
|
+
entries
|
|
1220
|
+
};
|
|
1221
|
+
return this.verified;
|
|
1222
|
+
}
|
|
1223
|
+
/** A refused transition can leave the projector half applied; discard it. */
|
|
1224
|
+
async guarded(run) {
|
|
1225
|
+
try {
|
|
1226
|
+
return run();
|
|
1227
|
+
} catch (error) {
|
|
1228
|
+
this.verified = null;
|
|
1229
|
+
throw error;
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
};
|
|
1141
1233
|
/** Verify and replay resolved immutable journal bytes without filesystem I/O. */
|
|
1142
1234
|
function replayLedgerText(text, path, codec) {
|
|
1143
1235
|
const state = emptyJournal(codec);
|
|
@@ -1246,6 +1338,6 @@ function readRange(fd, start, end) {
|
|
|
1246
1338
|
return bytes;
|
|
1247
1339
|
}
|
|
1248
1340
|
//#endregion
|
|
1249
|
-
export {
|
|
1341
|
+
export { probeAtomicFileLock as _, trustedHeadPathFor as a, readProjectorSnapshotFile as c, withLedgerFileLock as d, writeLedgerFileAtomically as f, AtomicFileLockError as g, mapConcurrentRange as h, readTrustedHeadFile as i, appendLedgerLine as l, mapConcurrent as m, MemoryLedgerJournal as n, verifyEntriesAgainstTrustedHead as o, Mutex as p, replayLedgerText as r, projectorSnapshotPathFor as s, FileLedgerJournal as t, tryWithLedgerFileLock as u, tryAcquireAtomicFileLock as v };
|
|
1250
1342
|
|
|
1251
|
-
//# sourceMappingURL=journal-
|
|
1343
|
+
//# sourceMappingURL=journal-DvKCvjty.js.map
|