@leostera/evalkit 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,111 +0,0 @@
1
- //#region ../core/src/upgrade.ts
2
- /** Report schema versions `evalkit upgrade` converts. */
3
- const UPGRADABLE_SCHEMA_VERSIONS = [
4
- 2,
5
- 3,
6
- 4,
7
- 5
8
- ];
9
- const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
10
- /**
11
- * A version 6 run status from an old trial's status and whether it passed.
12
- * Old `completed` meant "finished", passed or not; old `failed` meant it
13
- * could not finish, which version 6 calls `errored`.
14
- */
15
- function upgradeStatus(status, passed) {
16
- switch (status) {
17
- case "completed": return passed ? "passed" : "failed";
18
- case "failed": return "errored";
19
- case "cancelled": return "cancelled";
20
- default: return "running";
21
- }
22
- }
23
- /** A trajectory event with the old trial lifecycle events renamed. */
24
- function upgradeEvent(event) {
25
- if (!isRecord(event)) return event;
26
- if (event.kind === "trial-started") return {
27
- ...event,
28
- kind: "run-started"
29
- };
30
- if (event.kind === "trial-completed") return {
31
- ...event,
32
- kind: "run-completed"
33
- };
34
- return event;
35
- }
36
- /** A whole `trajectory.jsonl`, line by line; lines that are not JSON stay as they are. */
37
- function upgradeTrajectory(jsonl) {
38
- return jsonl.split("\n").map((line) => {
39
- if (!line.trim()) return line;
40
- try {
41
- return JSON.stringify(upgradeEvent(JSON.parse(line)));
42
- } catch {
43
- return line;
44
- }
45
- }).join("\n");
46
- }
47
- /**
48
- * A catalog (`catalog.json`, or its `catalog` list on its own) with each
49
- * scenario's `trialCount` and `policy.trials` replaced by `repeat`.
50
- */
51
- function upgradeCatalog(catalog) {
52
- const entry = (value) => {
53
- if (!isRecord(value)) return value;
54
- const { trialCount, policy, ...rest } = value;
55
- const { trials, ...keptPolicy } = isRecord(policy) ? policy : {};
56
- const repeat = typeof trialCount === "number" ? trialCount : typeof trials === "number" ? trials : value.repeat;
57
- return {
58
- ...rest,
59
- ...repeat === void 0 ? {} : { repeat },
60
- ...isRecord(policy) ? { policy: keptPolicy } : {}
61
- };
62
- };
63
- if (Array.isArray(catalog)) return catalog.map(entry);
64
- if (isRecord(catalog) && Array.isArray(catalog.catalog)) return {
65
- ...catalog,
66
- catalog: catalog.catalog.map(entry)
67
- };
68
- return catalog;
69
- }
70
- /** The fields of an old run manifest and trial manifest that a version 6 manifest keeps. */
71
- function upgradeManifest(input) {
72
- const { run, trial = {}, runId, batch, status } = input;
73
- const pick = (source, keys) => Object.fromEntries(keys.flatMap((key) => source[key] === void 0 ? [] : [[key, source[key]]]));
74
- return {
75
- schemaVersion: 6,
76
- runUri: `evalkit:run:${runId}`,
77
- ...pick(run, [
78
- "evalId",
79
- "suiteId",
80
- "parameters",
81
- "matrix",
82
- "aut",
83
- "startedAt",
84
- "source",
85
- "process"
86
- ]),
87
- ...pick(trial, [
88
- "evalId",
89
- "parameters",
90
- "matrix",
91
- "aut",
92
- "startedAt",
93
- "workspaceIgnore",
94
- "datasets"
95
- ]),
96
- ...batch ? { batch } : {},
97
- status
98
- };
99
- }
100
- /**
101
- * Scenario options from before 0.9 that plain JavaScript scenarios can still
102
- * carry. Ignoring them would silently change what runs, so both
103
- * `defineEval` and the runner reject them with the replacement.
104
- */
105
- function assertNoRemovedOptions(definition) {
106
- if ("fixtures" in definition) throw new Error(`Eval ${String(definition.id)} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
107
- if ("judge" in definition) throw new Error(`Eval ${String(definition.id)} sets judge, which moved to each rule: judge(name, { agent, rubric }), or judge in evalkit.config.ts for every judge (EvalKit 0.10)`);
108
- if (isRecord(definition.policy) && "trials" in definition.policy) throw new Error(`Eval ${String(definition.id)} sets policy.trials, which was replaced by repeat: ${JSON.stringify(definition.policy.trials)} (EvalKit 0.9)`);
109
- }
110
- //#endregion
111
- export { upgradeManifest as a, upgradeEvent as i, assertNoRemovedOptions as n, upgradeStatus as o, upgradeCatalog as r, upgradeTrajectory as s, UPGRADABLE_SCHEMA_VERSIONS as t };