@erdoai/cli 0.63.0 → 0.64.1

package/dist/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { readFileSync as readFileSync3 } from "fs";
+import { readFileSync as readFileSync4 } from "fs";
 import { basename } from "path";
 import { select } from "@inquirer/prompts";
 import { Command } from "commander";
@@ -418,7 +418,7 @@ var ErdoClient = class {
       `/v1/evals/suites/${encodeURIComponent(slug)}/cases/${encodeURIComponent(caseName)}`
     );
   }
-  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source) {
+  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride) {
     return this.request(
       "POST",
       `/v1/evals/suites/${encodeURIComponent(slug)}/run`,
@@ -427,10 +427,15 @@ var ErdoClient = class {
         commit_sha: commitSha,
         frontend_commit_sha: frontendCommitSha,
         backend_commit_sha: backendCommitSha,
-        source
+        source,
+        model_override: modelOverride
       }
     );
   }
+  getEvalLeaderboard(slug, days) {
+    const q = days && days > 0 ? `?days=${days}` : "";
+    return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
+  }
   getEvalRun(runID) {
     return this.request(
       "GET",
@@ -1328,9 +1333,17 @@ async function browserLogin() {
   return { token: data.access_token, me };
 }
 
+// src/input.ts
+import { readFileSync as readFileSync2 } from "fs";
+function readMaybeFile(value) {
+  if (!value) return void 0;
+  if (value.startsWith("@@")) return value.slice(1);
+  return value.startsWith("@") ? readFileSync2(value.slice(1), "utf8") : value;
+}
+
 // src/update.ts
 import { spawn as spawn2, spawnSync } from "child_process";
-import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync as writeFileSync2 } from "fs";
+import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
 import { dirname as dirname2 } from "path";
 var PKG = "@erdoai/cli";
 var MANUAL_HINT = `npm install -g ${PKG}@latest`;
@@ -1368,7 +1381,7 @@ async function fetchLatestVersion(timeoutMs) {
 function readVersionCache() {
   try {
     if (!existsSync2(CACHE_PATH)) return {};
-    const raw = JSON.parse(readFileSync2(CACHE_PATH, "utf8"));
+    const raw = JSON.parse(readFileSync3(CACHE_PATH, "utf8"));
     return raw && typeof raw === "object" ? raw : {};
   } catch {
     return {};
@@ -1436,7 +1449,7 @@ async function update(current) {
 }
 
 // src/index.ts
-var pkg = JSON.parse(readFileSync3(new URL("../package.json", import.meta.url), "utf8"));
+var pkg = JSON.parse(readFileSync4(new URL("../package.json", import.meta.url), "utf8"));
 function fail(err) {
   if (err instanceof ErdoApiError) {
     console.error(`Error ${err.status}: ${err.message}`);
@@ -2433,7 +2446,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
     }
   }
 );
-evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").action(async (slug, opts) => {
+evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").option("--model <model>", "pin the suite's agent to a model for this run (e.g. glm-5.3) \u2014 for model comparisons").action(async (slug, opts) => {
   try {
     const api = new ErdoClient();
     const { suite } = await api.getEvalSuite(slug);
@@ -2455,7 +2468,8 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
       opts.commitSha,
       opts.frontendCommitSha,
       opts.backendCommitSha,
-      opts.source
+      opts.source,
+      opts.model
     );
     console.log(`run_id: ${run_id}`);
     if (!opts.watch) return;
@@ -2481,6 +2495,44 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
     fail(e);
   }
 });
+evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).action(async (slug, opts) => {
+  try {
+    const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
+    if (res.models.length === 0) {
+      console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
+      return;
+    }
+    const modelCol = (m) => m.padEnd(18);
+    console.log(`
+${"Case".padEnd(34)}${res.models.map(modelCol).join("")}`);
+    console.log("-".repeat(34 + 18 * res.models.length));
+    for (const row of res.rows) {
+      const byModel = new Map(row.cells.map((c) => [c.model, c]));
+      let line = row.case_name.slice(0, 33).padEnd(34);
+      for (const m of res.models) {
+        const c = byModel.get(m);
+        line += c ? `${c.avg_score.toFixed(1)} (${Math.round(c.pass_rate * 100)}%)\xD7${c.n}`.padEnd(18) : "\xB7".padEnd(18);
+      }
+      console.log(line);
+    }
+    console.log(`
+${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`);
+    for (const stat of ["score", "pass", "cost", "secs"]) {
+      let line = `${"  " + stat}`.padEnd(34);
+      for (const s of res.summary) {
+        const v = stat === "score" ? s.avg_score.toFixed(2) : stat === "pass" ? `${Math.round(s.pass_rate * 100)}%` : stat === "cost" ? `$${(s.avg_cost_millicents / 1e5).toFixed(3)}/case` : `${(s.avg_duration_ms / 1e3).toFixed(0)}s`;
+        line += v.padEnd(18);
+      }
+      console.log(line);
+    }
+    console.log(
+      `
+Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
+    );
+  } catch (e) {
+    fail(e);
+  }
+});
 evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").action(async (runId, opts) => {
   try {
     const data = await new ErdoClient().getEvalRun(runId);
@@ -2599,14 +2651,14 @@ caseCmd.command("rm <slug> <caseName>").description("Archive a case from a suite
   }
 });
 var agentCmd = program.command("agent").description("Run agents");
-agentCmd.command("ask <question>").description("Ask the data-question agent (one-shot); prints the answer").option("-d, --datasets <csv>", "dataset slugs to scope to").option(
+agentCmd.command("ask <question>").description("Ask the data-question agent (one-shot); question accepts @path to a UTF-8 file").option("-d, --datasets <csv>", "dataset slugs to scope to").option(
   "--sync",
   "hold one long HTTP request open until the run finishes, instead of polling. Faster for quick questions; a proxy will time it out on any run over ~100s, losing the thread id while the run continues server-side"
 ).action(async (question, opts) => {
   try {
     const client = new ErdoClient();
     const res = await client.ask({
-      question,
+      question: readMaybeFile(question) ?? "",
       dataset_slugs: opts.datasets ? opts.datasets.split(",").map((s) => s.trim()) : void 0,
       async: !opts.sync
     });
@@ -2632,9 +2684,9 @@ thread: ${res.thread_id}`);
     fail(e);
   }
 });
-agentCmd.command("send <threadId> <message>").description("Send a message to a thread and print the agent's reply").option("-a, --agent <key>", "agent to invoke (e.g. erdo.artifact-builder)").option(
+agentCmd.command("send <threadId> <message>").description("Send a message to a thread and print the agent's reply; message accepts @path to a UTF-8 file").option("-a, --agent <key>", "agent to invoke (e.g. erdo.artifact-builder)").option(
   "--context <text>",
-  "application context for this turn; available to the agent but omitted from the visible user message"
+  "application context for this turn, or @path to a UTF-8 file; omitted from the visible user message"
 ).option(
   "--sync",
   "hold one long HTTP request open until the run finishes, instead of polling. Faster for quick questions; a proxy will time it out on any run over ~100s while the run continues server-side"
@@ -2643,8 +2695,8 @@ agentCmd.command("send <threadId> <message>").description("Send a message to a t
     const client = new ErdoClient();
     const baselineRunID = opts.sync ? void 0 : await client.latestRunID(threadId);
     const res = await client.sendMessage(threadId, {
-      message,
-      context: opts.context,
+      message: readMaybeFile(message) ?? "",
+      context: readMaybeFile(opts.context),
       agent_key: opts.agent,
       async: !opts.sync
     });
@@ -3552,10 +3604,6 @@ reviewsCmd.command("decide <id>").description(
   }
 );
 var pagesCmd = program.command("pages").description("Create and manage pages/artifacts");
-function readMaybeFile(v) {
-  if (!v) return void 0;
-  return v.startsWith("@") ? readFileSync3(v.slice(1), "utf8") : v;
-}
 function grantList(v) {
   if (v === void 0) return void 0;
   if (v.trim() === "[]") return [];
@@ -4095,7 +4143,7 @@ datasetsCmd.command("schema <slug>").alias("describe").description(
 var MAX_UPLOAD_BYTES = 20 * 1024 * 1024;
 datasetsCmd.command("upload <file>").description("Upload a file (CSV, Excel, JSON, ...) and create a dataset from it").option("-n, --name <name>", "display name for the dataset (defaults to the filename)").option("-d, --description <text>", "description shown to agents analyzing the dataset").action(async (file, opts) => {
   try {
-    const content = readFileSync3(file);
+    const content = readFileSync4(file);
     if (content.byteLength > MAX_UPLOAD_BYTES) {
       fail(
         new Error(
@@ -4524,7 +4572,7 @@ knowledgeCmd.command("attach <objectId> <file>").description(
     try {
       const body = opts.url ? { filename: basename(file), source_url: opts.url } : {
         filename: basename(file),
-        content_base64: readFileSync3(file).toString("base64")
+        content_base64: readFileSync4(file).toString("base64")
       };
       print(
         await new ErdoClient().attachKnowledgeAsset(objectId, {
@@ -4564,7 +4612,7 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
         fail(new Error("pass either --script-js or --script-file, not both"));
         return;
       }
-      const scriptJs = opts.scriptFile ? readFileSync3(opts.scriptFile, "utf8") : opts.scriptJs;
+      const scriptJs = opts.scriptFile ? readFileSync4(opts.scriptFile, "utf8") : opts.scriptJs;
       const body = {
         name: opts.name,
         description: opts.description,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@erdoai/cli",
-  "version": "0.63.0",
+  "version": "0.64.1",
   "description": "Erdo CLI — drive datasets, pages, and evals from the terminal or CI",
   "type": "module",
   "bin": {
@@ -35,6 +35,7 @@
   "scripts": {
     "build": "tsup src/index.ts --format esm --clean",
     "build:check": "tsc --noEmit",
+    "test": "node --import tsx --test src/input.test.ts",
     "dev": "tsx src/index.ts",
     "postinstall": "node scripts/postinstall.mjs",
     "prepublishOnly": "npm run build"