@leostera/evalkit 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +114 -1
- package/README.md +3 -2
- package/dist/cli.mjs +311 -92
- package/dist/dashboard/assets/index-CFJzcw3o.css +2 -0
- package/dist/dashboard/assets/{index-By1SJ1on.js → index-DgoX2x_M.js} +34 -31
- package/dist/dashboard/index.html +2 -2
- package/dist/{index-db-BbVblaHz.mjs → index-db-DI7xKMSX.mjs} +237 -47
- package/dist/index.d.mts +1141 -220
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +241 -52
- package/dist/runner.d.mts +782 -225
- package/dist/runner.d.mts.map +1 -1
- package/dist/runner.mjs +2 -2
- package/dist/share-worker.d.mts +1 -1
- package/dist/share-worker.d.mts.map +1 -1
- package/dist/share-worker.mjs +1 -1
- package/dist/share.d.mts +9 -6
- package/dist/share.d.mts.map +1 -1
- package/dist/share.mjs +2 -2
- package/dist/skills/evalkit/SKILL.md +3 -3
- package/dist/skills/evalkit-agents/SKILL.md +3 -3
- package/dist/skills/evalkit-datasets/SKILL.md +1 -1
- package/dist/skills/evalkit-evals/SKILL.md +1 -1
- package/dist/skills/evalkit-judges/SKILL.md +4 -4
- package/dist/skills/evalkit-results/SKILL.md +2 -2
- package/dist/{src-Bn6fnZnF.mjs → src-CXzbHAKN.mjs} +1 -1
- package/dist/{src-BrfIXfO2.mjs → src-D6Ax2t9b.mjs} +636 -220
- package/dist/src-Dc3GXacB.mjs +3637 -0
- package/dist/upgrade-rMsGzUEJ.mjs +230 -0
- package/dist/upgrade-transcript-C8DUykLd.mjs +402 -0
- package/package.json +2 -1
- package/dist/dashboard/assets/index-geXMXCx9.css +0 -2
- package/dist/src-CcdLZqFk.mjs +0 -1714
- package/dist/upgrade-60SIRius.mjs +0 -110
|
@@ -1,110 +0,0 @@
|
|
|
1
|
-
//#region ../core/src/upgrade.ts
|
|
2
|
-
/** Report schema versions `evalkit upgrade` converts. */
|
|
3
|
-
const UPGRADABLE_SCHEMA_VERSIONS = [
|
|
4
|
-
2,
|
|
5
|
-
3,
|
|
6
|
-
4,
|
|
7
|
-
5
|
|
8
|
-
];
|
|
9
|
-
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10
|
-
/**
|
|
11
|
-
* A version 6 run status from an old trial's status and whether it passed.
|
|
12
|
-
* Old `completed` meant "finished", passed or not; old `failed` meant it
|
|
13
|
-
* could not finish, which version 6 calls `errored`.
|
|
14
|
-
*/
|
|
15
|
-
function upgradeStatus(status, passed) {
|
|
16
|
-
switch (status) {
|
|
17
|
-
case "completed": return passed ? "passed" : "failed";
|
|
18
|
-
case "failed": return "errored";
|
|
19
|
-
case "cancelled": return "cancelled";
|
|
20
|
-
default: return "running";
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
/** A trajectory event with the old trial lifecycle events renamed. */
|
|
24
|
-
function upgradeEvent(event) {
|
|
25
|
-
if (!isRecord(event)) return event;
|
|
26
|
-
if (event.kind === "trial-started") return {
|
|
27
|
-
...event,
|
|
28
|
-
kind: "run-started"
|
|
29
|
-
};
|
|
30
|
-
if (event.kind === "trial-completed") return {
|
|
31
|
-
...event,
|
|
32
|
-
kind: "run-completed"
|
|
33
|
-
};
|
|
34
|
-
return event;
|
|
35
|
-
}
|
|
36
|
-
/** A whole `trajectory.jsonl`, line by line; lines that are not JSON stay as they are. */
|
|
37
|
-
function upgradeTrajectory(jsonl) {
|
|
38
|
-
return jsonl.split("\n").map((line) => {
|
|
39
|
-
if (!line.trim()) return line;
|
|
40
|
-
try {
|
|
41
|
-
return JSON.stringify(upgradeEvent(JSON.parse(line)));
|
|
42
|
-
} catch {
|
|
43
|
-
return line;
|
|
44
|
-
}
|
|
45
|
-
}).join("\n");
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* A catalog (`catalog.json`, or its `catalog` list on its own) with each
|
|
49
|
-
* scenario's `trialCount` and `policy.trials` replaced by `repeat`.
|
|
50
|
-
*/
|
|
51
|
-
function upgradeCatalog(catalog) {
|
|
52
|
-
const entry = (value) => {
|
|
53
|
-
if (!isRecord(value)) return value;
|
|
54
|
-
const { trialCount, policy, ...rest } = value;
|
|
55
|
-
const { trials, ...keptPolicy } = isRecord(policy) ? policy : {};
|
|
56
|
-
const repeat = typeof trialCount === "number" ? trialCount : typeof trials === "number" ? trials : value.repeat;
|
|
57
|
-
return {
|
|
58
|
-
...rest,
|
|
59
|
-
...repeat === void 0 ? {} : { repeat },
|
|
60
|
-
...isRecord(policy) ? { policy: keptPolicy } : {}
|
|
61
|
-
};
|
|
62
|
-
};
|
|
63
|
-
if (Array.isArray(catalog)) return catalog.map(entry);
|
|
64
|
-
if (isRecord(catalog) && Array.isArray(catalog.catalog)) return {
|
|
65
|
-
...catalog,
|
|
66
|
-
catalog: catalog.catalog.map(entry)
|
|
67
|
-
};
|
|
68
|
-
return catalog;
|
|
69
|
-
}
|
|
70
|
-
/** The fields of an old run manifest and trial manifest that a version 6 manifest keeps. */
|
|
71
|
-
function upgradeManifest(input) {
|
|
72
|
-
const { run, trial = {}, runId, batch, status } = input;
|
|
73
|
-
const pick = (source, keys) => Object.fromEntries(keys.flatMap((key) => source[key] === void 0 ? [] : [[key, source[key]]]));
|
|
74
|
-
return {
|
|
75
|
-
schemaVersion: 6,
|
|
76
|
-
runUri: `evalkit:run:${runId}`,
|
|
77
|
-
...pick(run, [
|
|
78
|
-
"evalId",
|
|
79
|
-
"suiteId",
|
|
80
|
-
"parameters",
|
|
81
|
-
"matrix",
|
|
82
|
-
"aut",
|
|
83
|
-
"startedAt",
|
|
84
|
-
"source",
|
|
85
|
-
"process"
|
|
86
|
-
]),
|
|
87
|
-
...pick(trial, [
|
|
88
|
-
"evalId",
|
|
89
|
-
"parameters",
|
|
90
|
-
"matrix",
|
|
91
|
-
"aut",
|
|
92
|
-
"startedAt",
|
|
93
|
-
"workspaceIgnore",
|
|
94
|
-
"datasets"
|
|
95
|
-
]),
|
|
96
|
-
...batch ? { batch } : {},
|
|
97
|
-
status
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Scenario options from before 0.9 that plain JavaScript scenarios can still
|
|
102
|
-
* carry. Ignoring them would silently change what runs, so both
|
|
103
|
-
* `defineEval` and the runner reject them with the replacement.
|
|
104
|
-
*/
|
|
105
|
-
function assertNoRemovedOptions(definition) {
|
|
106
|
-
if ("fixtures" in definition) throw new Error(`Eval ${String(definition.id)} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
|
|
107
|
-
if (isRecord(definition.policy) && "trials" in definition.policy) throw new Error(`Eval ${String(definition.id)} sets policy.trials, which was replaced by repeat: ${JSON.stringify(definition.policy.trials)} (EvalKit 0.9)`);
|
|
108
|
-
}
|
|
109
|
-
//#endregion
|
|
110
|
-
export { upgradeManifest as a, upgradeEvent as i, assertNoRemovedOptions as n, upgradeStatus as o, upgradeCatalog as r, upgradeTrajectory as s, UPGRADABLE_SCHEMA_VERSIONS as t };
|