@erdoai/cli 0.62.0 → 0.64.0

package/dist/index.js

@@ -269,6 +269,15 @@ var ErdoClient = class {
   setAutonomyMode(mode) {
     return this.request("PUT", "/v1/autonomy-mode", { mode });
   }
+  // Personal approval settings — which risk classes of the caller's OWN agent
+  // runs still raise a card in the active org. gates null = no personal
+  // setting (org default); [] = nothing asks; otherwise the listed classes ask.
+  getApprovalSettings() {
+    return this.request("GET", "/v1/approval-settings");
+  }
+  setApprovalSettings(gates) {
+    return this.request("PUT", "/v1/approval-settings", { gates });
+  }
   // --- Manager accounts: operate many client (managed) orgs with ONE manager key ---
   // The manager is your active org; you must be an admin/owner of it. A managed org
   // is a client tenant you provision and operate. The manager key (createManagerKey)
@@ -409,7 +418,7 @@ var ErdoClient = class {
       `/v1/evals/suites/${encodeURIComponent(slug)}/cases/${encodeURIComponent(caseName)}`
     );
   }
-  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source) {
+  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride) {
     return this.request(
       "POST",
       `/v1/evals/suites/${encodeURIComponent(slug)}/run`,
@@ -418,10 +427,15 @@ var ErdoClient = class {
         commit_sha: commitSha,
         frontend_commit_sha: frontendCommitSha,
         backend_commit_sha: backendCommitSha,
-        source
+        source,
+        model_override: modelOverride
       }
     );
   }
+  getEvalLeaderboard(slug, days) {
+    const q = days && days > 0 ? `?days=${days}` : "";
+    return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
+  }
   getEvalRun(runID) {
     return this.request(
       "GET",
@@ -2424,7 +2438,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
     }
   }
 );
-evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").action(async (slug, opts) => {
+evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").option("--model <model>", "pin the suite's agent to a model for this run (e.g. glm-5.3) \u2014 for model comparisons").action(async (slug, opts) => {
   try {
     const api = new ErdoClient();
     const { suite } = await api.getEvalSuite(slug);
@@ -2446,7 +2460,8 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
       opts.commitSha,
       opts.frontendCommitSha,
       opts.backendCommitSha,
-      opts.source
+      opts.source,
+      opts.model
     );
     console.log(`run_id: ${run_id}`);
     if (!opts.watch) return;
@@ -2472,6 +2487,44 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
     fail(e);
   }
 });
+evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).action(async (slug, opts) => {
+  try {
+    const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
+    if (res.models.length === 0) {
+      console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
+      return;
+    }
+    const modelCol = (m) => m.padEnd(18);
+    console.log(`
+${"Case".padEnd(34)}${res.models.map(modelCol).join("")}`);
+    console.log("-".repeat(34 + 18 * res.models.length));
+    for (const row of res.rows) {
+      const byModel = new Map(row.cells.map((c) => [c.model, c]));
+      let line = row.case_name.slice(0, 33).padEnd(34);
+      for (const m of res.models) {
+        const c = byModel.get(m);
+        line += c ? `${c.avg_score.toFixed(1)} (${Math.round(c.pass_rate * 100)}%)\xD7${c.n}`.padEnd(18) : "\xB7".padEnd(18);
+      }
+      console.log(line);
+    }
+    console.log(`
+${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`);
+    for (const stat of ["score", "pass", "cost", "secs"]) {
+      let line = `${"  " + stat}`.padEnd(34);
+      for (const s of res.summary) {
+        const v = stat === "score" ? s.avg_score.toFixed(2) : stat === "pass" ? `${Math.round(s.pass_rate * 100)}%` : stat === "cost" ? `$${(s.avg_cost_millicents / 1e5).toFixed(3)}/case` : `${(s.avg_duration_ms / 1e3).toFixed(0)}s`;
+        line += v.padEnd(18);
+      }
+      console.log(line);
+    }
+    console.log(
+      `
+Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
+    );
+  } catch (e) {
+    fail(e);
+  }
+});
 evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").action(async (runId, opts) => {
   try {
     const data = await new ErdoClient().getEvalRun(runId);
@@ -3109,6 +3162,50 @@ ${listing}`);
     }
   }
 );
+approvalsCmd.command("settings [mode]").description(
+  "Show or set YOUR approval settings: which classes of your agents' actions still ask. mode: safe (auto-approve page work + bookkeeping; spend and destructive still ask) | all (approve everything) | reset (back to the org default)"
+).option(
+  "--gates <classes>",
+  "explicit comma-separated gate list, e.g. spend,destructive (overrides <mode>)"
+).action(async (mode, opts = {}) => {
+  try {
+    const client = new ErdoClient();
+    let gates;
+    if (opts.gates !== void 0) {
+      gates = opts.gates.split(",").map((g) => g.trim()).filter(Boolean);
+    } else {
+      switch (mode) {
+        case void 0:
+          break;
+        // show
+        case "safe":
+          gates = ["spend", "destructive"];
+          break;
+        case "all":
+          gates = [];
+          break;
+        case "reset":
+          gates = null;
+          break;
+        default:
+          throw new Error(`unknown mode ${mode} \u2014 use safe | all | reset, or --gates spend,destructive`);
+      }
+    }
+    if (gates !== void 0) {
+      await client.setApprovalSettings(gates);
+    }
+    const { gates: current } = await client.getApprovalSettings();
+    if (current === null) {
+      console.log("Approval settings: org default (every gated action asks, subject to the org's own gates and standing policies)");
+    } else if (current.length === 0) {
+      console.log("Approval settings: approve everything \u2014 no class of your agents' actions asks (a standing always-require policy still applies)");
+    } else {
+      console.log(`Approval settings: ${current.join(", ")} still ask; the rest of your agents' actions auto-approve`);
+    }
+  } catch (e) {
+    fail(e);
+  }
+});
 var attnCmd = program.command("attention").description("The attention feed \u2014 digests, choices, escalations awaiting a human");
 attnCmd.command("list").description("List attention items").option("--status <status...>", "open | answered | dismissed | expired").option("--open", "shorthand for --status open").option("--engine-actions", "only engine-generated items").option("--item <idOrSlug>", "read one item: its slug or its uuid").option("-n, --limit <n>", "max items", (v) => parseInt(v, 10)).option("--offset <n>", "pagination offset", (v) => parseInt(v, 10)).action(
   async (opts) => {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@erdoai/cli",
-  "version": "0.62.0",
-  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
+  "version": "0.64.0",
+  "description": "Erdo CLI — drive datasets, pages, and evals from the terminal or CI",
   "type": "module",
   "bin": {
     "erdo": "dist/index.js"
@@ -52,4 +52,4 @@
   "overrides": {
     "esbuild": "^0.28.1"
   }
-}
+}