@erdoai/cli 0.189.0 → 0.190.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +62 -3
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -122,6 +122,35 @@ function parseCSV(text) {
122
122
  return rows;
123
123
  }
124
124
 
125
+ // src/eval-compare.ts
126
+ function evalResultScored(r) {
127
+ return !r.agent_error && r.failure_classification !== "judge_error";
128
+ }
129
+ function compareEvalRuns(before, after, cases = []) {
130
+ const nameByID = new Map(cases.map((c) => [c.id, c.name]));
131
+ const beforeByCase = new Map(before.map((r) => [r.case_id, r]));
132
+ const afterByCase = new Map(after.map((r) => [r.case_id, r]));
133
+ const ids = Array.from(/* @__PURE__ */ new Set([...beforeByCase.keys(), ...afterByCase.keys()]));
134
+ const rows = ids.map((id) => {
135
+ const b = beforeByCase.get(id);
136
+ const a = afterByCase.get(id);
137
+ const bScore = b && evalResultScored(b) ? b.score : null;
138
+ const aScore = a && evalResultScored(a) ? a.score : null;
139
+ let flip = "";
140
+ if (bScore === null || aScore === null) flip = "unscored";
141
+ else if (!b.passed && a.passed) flip = "now passes";
142
+ else if (b.passed && !a.passed) flip = "now fails";
143
+ return {
144
+ case: nameByID.get(id) ?? id,
145
+ before: bScore,
146
+ after: aScore,
147
+ change: bScore !== null && aScore !== null ? Math.round((aScore - bScore) * 100) / 100 : null,
148
+ flip
149
+ };
150
+ });
151
+ return rows.sort((x, y) => (y.change ?? -Infinity) - (x.change ?? -Infinity));
152
+ }
153
+
125
154
  // src/index.ts
126
155
  import { password, select } from "@inquirer/prompts";
127
156
  import { Argument, Command as Command2, Option as Option2 } from "commander";
@@ -2867,6 +2896,7 @@ function summariseResults(results, cases = []) {
2867
2896
  for (const r of results) {
2868
2897
  let status;
2869
2898
  if (r.agent_error) status = "ERROR";
2899
+ else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
2870
2900
  else if (r.passed) status = "PASS";
2871
2901
  else status = "FAIL";
2872
2902
  const name = nameByID.get(r.case_id) ?? r.case_id;
@@ -2877,6 +2907,13 @@ function summariseResults(results, cases = []) {
2877
2907
  }
2878
2908
  }
2879
2909
  }
2910
+ function printEvalComparison(rows) {
2911
+ const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
2912
+ for (const r of rows) {
2913
+ const change = r.change === null ? "" : `${r.change >= 0 ? "+" : ""}${r.change.toFixed(2)}`;
2914
+ console.log(` ${fmt(r.before)} -> ${fmt(r.after)} ${change.padEnd(6)} ${r.case}${r.flip ? ` (${r.flip})` : ""}`);
2915
+ }
2916
+ }
2880
2917
  function printEvalRunAttribution(run) {
2881
2918
  console.log(`source: ${run.triggered_by}`);
2882
2919
  if (run.commit_sha) console.log(`commit: ${run.commit_sha}`);
@@ -4726,14 +4763,36 @@ Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; ag
4726
4763
  fail(e);
4727
4764
  }
4728
4765
  });
4729
- evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").action(async (runId, opts) => {
4766
+ evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").option(
4767
+ "--compare <baselineRunId>",
4768
+ "compare against an earlier run of the same suite: per-case score change and which cases flipped"
4769
+ ).action(async (runId, opts) => {
4730
4770
  try {
4731
- const data = await new ErdoClient().getEvalRun(runId);
4732
- if (opts.json) {
4771
+ const client = new ErdoClient();
4772
+ const data = await client.getEvalRun(runId);
4773
+ if (opts.json && !opts.compare) {
4733
4774
  print(data);
4734
4775
  return;
4735
4776
  }
4736
4777
  const { run, results, cases } = data;
4778
+ if (opts.compare) {
4779
+ const baseline = await client.getEvalRun(opts.compare);
4780
+ if (baseline.run.suite_id !== run.suite_id) {
4781
+ throw new Error(
4782
+ `run ${opts.compare} belongs to a different suite; compare two runs of the same suite (erdo eval runs --suite <slug>)`
4783
+ );
4784
+ }
4785
+ const comparison = compareEvalRuns(baseline.results, results, cases.length ? cases : baseline.cases);
4786
+ if (opts.json) {
4787
+ print(comparison);
4788
+ return;
4789
+ }
4790
+ console.log(
4791
+ `avg ${baseline.run.avg_score.toFixed(2)} -> ${run.avg_score.toFixed(2)}, passed ${baseline.run.passed_cases} -> ${run.passed_cases} of ${run.total_cases}`
4792
+ );
4793
+ printEvalComparison(comparison);
4794
+ return;
4795
+ }
4737
4796
  console.log(
4738
4797
  `${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_score.toFixed(2)}`
4739
4798
  );
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.189.0",
3
+ "version": "0.190.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {