@erdoai/cli 0.188.0 → 0.190.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +62 -3
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -122,6 +122,35 @@ function parseCSV(text) {
|
|
|
122
122
|
return rows;
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
// src/eval-compare.ts
|
|
126
|
+
function evalResultScored(r) {
|
|
127
|
+
return !r.agent_error && r.failure_classification !== "judge_error";
|
|
128
|
+
}
|
|
129
|
+
function compareEvalRuns(before, after, cases = []) {
|
|
130
|
+
const nameByID = new Map(cases.map((c) => [c.id, c.name]));
|
|
131
|
+
const beforeByCase = new Map(before.map((r) => [r.case_id, r]));
|
|
132
|
+
const afterByCase = new Map(after.map((r) => [r.case_id, r]));
|
|
133
|
+
const ids = Array.from(/* @__PURE__ */ new Set([...beforeByCase.keys(), ...afterByCase.keys()]));
|
|
134
|
+
const rows = ids.map((id) => {
|
|
135
|
+
const b = beforeByCase.get(id);
|
|
136
|
+
const a = afterByCase.get(id);
|
|
137
|
+
const bScore = b && evalResultScored(b) ? b.score : null;
|
|
138
|
+
const aScore = a && evalResultScored(a) ? a.score : null;
|
|
139
|
+
let flip = "";
|
|
140
|
+
if (bScore === null || aScore === null) flip = "unscored";
|
|
141
|
+
else if (!b.passed && a.passed) flip = "now passes";
|
|
142
|
+
else if (b.passed && !a.passed) flip = "now fails";
|
|
143
|
+
return {
|
|
144
|
+
case: nameByID.get(id) ?? id,
|
|
145
|
+
before: bScore,
|
|
146
|
+
after: aScore,
|
|
147
|
+
change: bScore !== null && aScore !== null ? Math.round((aScore - bScore) * 100) / 100 : null,
|
|
148
|
+
flip
|
|
149
|
+
};
|
|
150
|
+
});
|
|
151
|
+
return rows.sort((x, y) => (y.change ?? -Infinity) - (x.change ?? -Infinity));
|
|
152
|
+
}
|
|
153
|
+
|
|
125
154
|
// src/index.ts
|
|
126
155
|
import { password, select } from "@inquirer/prompts";
|
|
127
156
|
import { Argument, Command as Command2, Option as Option2 } from "commander";
|
|
@@ -2867,6 +2896,7 @@ function summariseResults(results, cases = []) {
|
|
|
2867
2896
|
for (const r of results) {
|
|
2868
2897
|
let status;
|
|
2869
2898
|
if (r.agent_error) status = "ERROR";
|
|
2899
|
+
else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
|
|
2870
2900
|
else if (r.passed) status = "PASS";
|
|
2871
2901
|
else status = "FAIL";
|
|
2872
2902
|
const name = nameByID.get(r.case_id) ?? r.case_id;
|
|
@@ -2877,6 +2907,13 @@ function summariseResults(results, cases = []) {
|
|
|
2877
2907
|
}
|
|
2878
2908
|
}
|
|
2879
2909
|
}
|
|
2910
|
+
function printEvalComparison(rows) {
|
|
2911
|
+
const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
|
|
2912
|
+
for (const r of rows) {
|
|
2913
|
+
const change = r.change === null ? "" : `${r.change >= 0 ? "+" : ""}${r.change.toFixed(2)}`;
|
|
2914
|
+
console.log(` ${fmt(r.before)} -> ${fmt(r.after)} ${change.padEnd(6)} ${r.case}${r.flip ? ` (${r.flip})` : ""}`);
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2880
2917
|
function printEvalRunAttribution(run) {
|
|
2881
2918
|
console.log(`source: ${run.triggered_by}`);
|
|
2882
2919
|
if (run.commit_sha) console.log(`commit: ${run.commit_sha}`);
|
|
@@ -4726,14 +4763,36 @@ Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; ag
|
|
|
4726
4763
|
fail(e);
|
|
4727
4764
|
}
|
|
4728
4765
|
});
|
|
4729
|
-
evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").
|
|
4766
|
+
evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").option(
|
|
4767
|
+
"--compare <baselineRunId>",
|
|
4768
|
+
"compare against an earlier run of the same suite: per-case score change and which cases flipped"
|
|
4769
|
+
).action(async (runId, opts) => {
|
|
4730
4770
|
try {
|
|
4731
|
-
const
|
|
4732
|
-
|
|
4771
|
+
const client = new ErdoClient();
|
|
4772
|
+
const data = await client.getEvalRun(runId);
|
|
4773
|
+
if (opts.json && !opts.compare) {
|
|
4733
4774
|
print(data);
|
|
4734
4775
|
return;
|
|
4735
4776
|
}
|
|
4736
4777
|
const { run, results, cases } = data;
|
|
4778
|
+
if (opts.compare) {
|
|
4779
|
+
const baseline = await client.getEvalRun(opts.compare);
|
|
4780
|
+
if (baseline.run.suite_id !== run.suite_id) {
|
|
4781
|
+
throw new Error(
|
|
4782
|
+
`run ${opts.compare} belongs to a different suite; compare two runs of the same suite (erdo eval runs --suite <slug>)`
|
|
4783
|
+
);
|
|
4784
|
+
}
|
|
4785
|
+
const comparison = compareEvalRuns(baseline.results, results, cases.length ? cases : baseline.cases);
|
|
4786
|
+
if (opts.json) {
|
|
4787
|
+
print(comparison);
|
|
4788
|
+
return;
|
|
4789
|
+
}
|
|
4790
|
+
console.log(
|
|
4791
|
+
`avg ${baseline.run.avg_score.toFixed(2)} -> ${run.avg_score.toFixed(2)}, passed ${baseline.run.passed_cases} -> ${run.passed_cases} of ${run.total_cases}`
|
|
4792
|
+
);
|
|
4793
|
+
printEvalComparison(comparison);
|
|
4794
|
+
return;
|
|
4795
|
+
}
|
|
4737
4796
|
console.log(
|
|
4738
4797
|
`${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_score.toFixed(2)}`
|
|
4739
4798
|
);
|