@erdoai/cli 0.63.0 → 0.64.0

package/dist/index.js

@@ -418,7 +418,7 @@ var ErdoClient = class {
       `/v1/evals/suites/${encodeURIComponent(slug)}/cases/${encodeURIComponent(caseName)}`
     );
   }
-  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source) {
+  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride) {
     return this.request(
       "POST",
       `/v1/evals/suites/${encodeURIComponent(slug)}/run`,
@@ -427,10 +427,15 @@ var ErdoClient = class {
         commit_sha: commitSha,
         frontend_commit_sha: frontendCommitSha,
         backend_commit_sha: backendCommitSha,
-        source
+        source,
+        model_override: modelOverride
       }
     );
   }
+  getEvalLeaderboard(slug, days) {
+    const q = days && days > 0 ? `?days=${days}` : "";
+    return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
+  }
   getEvalRun(runID) {
     return this.request(
       "GET",
@@ -2433,7 +2438,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
     }
   }
 );
-evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").action(async (slug, opts) => {
+evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").option("--model <model>", "pin the suite's agent to a model for this run (e.g. glm-5.3) \u2014 for model comparisons").action(async (slug, opts) => {
   try {
     const api = new ErdoClient();
     const { suite } = await api.getEvalSuite(slug);
@@ -2455,7 +2460,8 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
       opts.commitSha,
       opts.frontendCommitSha,
       opts.backendCommitSha,
-      opts.source
+      opts.source,
+      opts.model
     );
     console.log(`run_id: ${run_id}`);
     if (!opts.watch) return;
@@ -2481,6 +2487,44 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
     fail(e);
   }
 });
+evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).action(async (slug, opts) => {
+  try {
+    const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
+    if (res.models.length === 0) {
+      console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
+      return;
+    }
+    const modelCol = (m) => m.padEnd(18);
+    console.log(`
+${"Case".padEnd(34)}${res.models.map(modelCol).join("")}`);
+    console.log("-".repeat(34 + 18 * res.models.length));
+    for (const row of res.rows) {
+      const byModel = new Map(row.cells.map((c) => [c.model, c]));
+      let line = row.case_name.slice(0, 33).padEnd(34);
+      for (const m of res.models) {
+        const c = byModel.get(m);
+        line += c ? `${c.avg_score.toFixed(1)} (${Math.round(c.pass_rate * 100)}%)\xD7${c.n}`.padEnd(18) : "\xB7".padEnd(18);
+      }
+      console.log(line);
+    }
+    console.log(`
+${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`);
+    for (const stat of ["score", "pass", "cost", "secs"]) {
+      let line = `${"  " + stat}`.padEnd(34);
+      for (const s of res.summary) {
+        const v = stat === "score" ? s.avg_score.toFixed(2) : stat === "pass" ? `${Math.round(s.pass_rate * 100)}%` : stat === "cost" ? `$${(s.avg_cost_millicents / 1e5).toFixed(3)}/case` : `${(s.avg_duration_ms / 1e3).toFixed(0)}s`;
+        line += v.padEnd(18);
+      }
+      console.log(line);
+    }
+    console.log(
+      `
+Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
+    );
+  } catch (e) {
+    fail(e);
+  }
+});
 evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").action(async (runId, opts) => {
   try {
     const data = await new ErdoClient().getEvalRun(runId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@erdoai/cli",
-  "version": "0.63.0",
+  "version": "0.64.0",
   "description": "Erdo CLI — drive datasets, pages, and evals from the terminal or CI",
   "type": "module",
   "bin": {