@erdoai/cli 0.55.1 → 0.55.3

package/dist/index.js

@@ -362,11 +362,17 @@ var ErdoClient = class {
       `/v1/evals/suites/${encodeURIComponent(slug)}/cases/${encodeURIComponent(caseName)}`
     );
   }
-  runEvalSuite(slug, concurrency) {
+  runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source) {
     return this.request(
       "POST",
       `/v1/evals/suites/${encodeURIComponent(slug)}/run`,
-      { concurrency: concurrency ?? 0 }
+      {
+        concurrency: concurrency ?? 0,
+        commit_sha: commitSha,
+        frontend_commit_sha: frontendCommitSha,
+        backend_commit_sha: backendCommitSha,
+        source
+      }
     );
   }
   getEvalRun(runID) {
@@ -375,6 +381,14 @@ var ErdoClient = class {
       `/v1/evals/runs/${encodeURIComponent(runID)}`
     );
   }
+  getEvalWidgetCanary() {
+    return this.request("GET", "/v1/evals/widget-canary");
+  }
+  configureEvalWidgetCanary(widget) {
+    return this.request("PUT", "/v1/evals/widget-canary", {
+      widget
+    });
+  }
   // --- projects ---
   listProjects(params) {
     const q = new URLSearchParams();
@@ -1384,6 +1398,20 @@ function summariseResults(results, cases = []) {
     }
   }
 }
+function printEvalRunAttribution(run) {
+  console.log(`source: ${run.triggered_by}`);
+  if (run.commit_sha) console.log(`commit: ${run.commit_sha}`);
+  if (run.frontend_commit_sha) console.log(`frontend commit: ${run.frontend_commit_sha}`);
+  if (run.backend_commit_sha) console.log(`backend commit: ${run.backend_commit_sha}`);
+}
+function compactEvalRunAttribution(run) {
+  const revisions = [
+    run.commit_sha ? `commit ${run.commit_sha.slice(0, 12)}` : void 0,
+    run.frontend_commit_sha ? `frontend ${run.frontend_commit_sha.slice(0, 12)}` : void 0,
+    run.backend_commit_sha ? `backend ${run.backend_commit_sha.slice(0, 12)}` : void 0
+  ].filter((part) => part !== void 0);
+  return [run.triggered_by, ...revisions].join("	");
+}
 var program = new Command();
 program.name("erdo").description("Erdo CLI").version(pkg.version).option("--org <idOrSlug>", "org to target for this command (overrides the active org)").option("--project <uuid>", "project context for this command (must belong to the active org)");
 program.hook("preAction", (thisCommand) => {
@@ -2228,18 +2256,41 @@ function caseScoringPayload(opts) {
   }
   return out;
 }
-evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold 0-5", (v) => parseFloat(v)).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
+function parseEvalThreshold(value) {
+  if (!/^(?:\d+(?:\.\d*)?|\.\d+)$/.test(value.trim())) {
+    throw new Error("threshold must be greater than 0 and at most 5");
+  }
+  const threshold = Number(value);
+  if (!Number.isFinite(threshold) || threshold <= 0 || threshold > 5) {
+    throw new Error("threshold must be greater than 0 and at most 5");
+  }
+  return threshold;
+}
+function parseStrictBoolean(value) {
+  switch (value.trim().toLowerCase()) {
+    case "true":
+    case "1":
+    case "yes":
+      return true;
+    case "false":
+    case "0":
+    case "no":
+      return false;
+    default:
+      throw new Error("value must be true or false");
+  }
+}
+evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
   async (slug, opts) => {
     try {
-      const parseBool = (v) => v === void 0 ? void 0 : v === "true" || v === "1" || v === "yes";
       const res = await new ErdoClient().updateEvalSuite(slug, {
         name: opts.name,
         description: opts.description,
         agent_key: opts.agent,
         judge_model: opts.judge,
         pass_threshold: opts.threshold,
-        evaluate_artifact: parseBool(opts.evaluateArtifact),
-        cron_enabled: parseBool(opts.cron)
+        evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
+        cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
       });
       print(res);
     } catch (e) {
@@ -2247,7 +2298,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
     }
   }
 );
-evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold 0-5", (v) => parseFloat(v)).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
+evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
   acc.push(v);
   return acc;
 }, []).action(
@@ -2271,7 +2322,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
     }
   }
 );
-evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).action(async (slug, opts) => {
+evalCmd.command("run <slug>").description("Run a suite; --watch polls until it completes").option("-w, --watch", "poll until the run finishes and print results").option("-c, --concurrency <n>", "parallel cases", (v) => parseInt(v, 10)).option("--commit-sha <sha>", "full 40-character Git commit SHA to associate with the run").option("--frontend-commit-sha <sha>", "frontend revision in the evaluated production snapshot").option("--backend-commit-sha <sha>", "backend revision in the evaluated production snapshot").option("--source <source>", "staff-only run source: post_deploy, nightly, or rollback").action(async (slug, opts) => {
   try {
     const api = new ErdoClient();
     const { suite } = await api.getEvalSuite(slug);
@@ -2287,7 +2338,14 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
       );
       process.exit(1);
     }
-    const { run_id } = await api.runEvalSuite(slug, opts.concurrency);
+    const { run_id } = await api.runEvalSuite(
+      slug,
+      opts.concurrency,
+      opts.commitSha,
+      opts.frontendCommitSha,
+      opts.backendCommitSha,
+      opts.source
+    );
     console.log(`run_id: ${run_id}`);
     if (!opts.watch) return;
     for (; ; ) {
@@ -2301,6 +2359,7 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
         `
 ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_score.toFixed(2)}`
       );
+      printEvalRunAttribution(run);
       summariseResults(results, cases);
       if (run.status !== "completed" || run.passed_cases < run.total_cases) {
         process.exitCode = 1;
@@ -2322,6 +2381,7 @@ evalCmd.command("results <runId>").description("Show a run's results").option("-
     console.log(
       `${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_score.toFixed(2)}`
     );
+    printEvalRunAttribution(run);
     summariseResults(results, cases);
   } catch (e) {
     fail(e);
@@ -2332,9 +2392,41 @@ evalCmd.command("runs").description("List recent runs").option("-s, --suite <slu
     const { runs } = await new ErdoClient().listEvalRuns(opts.suite, opts.limit);
     for (const r of runs) {
       console.log(
-        `${r.id}	${r.status}	${r.passed_cases}/${r.total_cases}	avg ${r.avg_score.toFixed(2)}`
+        `${r.id}	${r.status}	${r.passed_cases}/${r.total_cases}	avg ${r.avg_score.toFixed(2)}	${compactEvalRunAttribution(r)}`
+      );
+    }
+  } catch (e) {
+    fail(e);
+  }
+});
+var evalCanaryCmd = evalCmd.command("canary").description("Show or configure the current organization's deployed-experience widget canary (staff only)");
+evalCanaryCmd.command("show").description("Show the widget designated as this organization's eval canary").action(async () => {
+  try {
+    print(await new ErdoClient().getEvalWidgetCanary());
+  } catch (e) {
+    fail(e);
+  }
+});
+evalCanaryCmd.command("set <widget>").description("Designate a widget by unique name or public key in the pinned organization").action(async (widget) => {
+  try {
+    if (!process.env.ERDO_ORG) {
+      throw new Error(
+        "Refusing to change an eval canary without a pinned org. Re-run as: erdo --org <id|slug> eval canary set <widget-name-or-public-key>"
       );
     }
+    print(await new ErdoClient().configureEvalWidgetCanary(widget));
+  } catch (e) {
+    fail(e);
+  }
+});
+evalCanaryCmd.command("clear").description("Clear the pinned organization's eval widget canary").action(async () => {
+  try {
+    if (!process.env.ERDO_ORG) {
+      throw new Error(
+        "Refusing to clear an eval canary without a pinned org. Re-run as: erdo --org <id|slug> eval canary clear"
+      );
+    }
+    print(await new ErdoClient().configureEvalWidgetCanary(""));
   } catch (e) {
     fail(e);
   }
@@ -2345,6 +2437,9 @@ caseCmd.command("add <slug>").description("Add a case to a suite").requiredOptio
   "a turn to run BEFORE input, same thread/agent (repeatable; e.g. 'create a voice widget for X')",
   collect,
   []
+).option(
+  "--target-type <type>",
+  "closed deployed target: live_page_fleet, widget_fleet_live, widget_text_canary, widget_voice_canary, or widget_video_canary"
 ).option("--rubric <json>", 'LLM rubric: JSON array of {"criterion","weight"}').option("--evaluator <json>", "evaluator JSON {type,weight,...} (repeatable)", collect, []).option("--tags <csv>", "comma-separated tags").action(
   async (slug, opts) => {
     try {
@@ -2352,6 +2447,7 @@ caseCmd.command("add <slug>").description("Add a case to a suite").requiredOptio
         name: opts.name,
         input: opts.input,
         setup_messages: opts.setup,
+        target_type: opts.targetType,
         ...caseScoringPayload(opts),
         tags: opts.tags ? opts.tags.split(",").map((t) => t.trim()) : []
       });
@@ -2361,13 +2457,19 @@ caseCmd.command("add <slug>").description("Add a case to a suite").requiredOptio
     }
   }
 );
-caseCmd.command("update <slug> <caseName>").description("Replace a case's brief, scoring, and tags").requiredOption("--input <brief>", "replacement brief/prompt (the evaluated turn)").option("--setup <msg>", "replacement pre-turn run before input (repeatable; omit for none)", collect, []).option("--rubric <json>", "replacement LLM rubric JSON").option("--evaluator <json>", "evaluator JSON {type,weight,...} (repeatable)", collect, []).option("--tags <csv>", "comma-separated tags").action(
+caseCmd.command("update <slug> <caseName>").description("Replace a case's brief, scoring, and tags").requiredOption("--input <brief>", "replacement brief/prompt (the evaluated turn)").option("--setup <msg>", "replacement pre-turn run before input (repeatable; omit to preserve)", collect, []).option(
+  "--target-type <type>",
+  "replacement deployed target: live_page_fleet, widget_fleet_live, widget_text_canary, widget_voice_canary, or widget_video_canary (omit to preserve)"
+).option("--clear-setup", "remove all existing setup turns").option("--clear-target", "remove the deployed target and return to ordinary agent execution").option("--rubric <json>", "replacement LLM rubric JSON").option("--evaluator <json>", "evaluator JSON {type,weight,...} (repeatable)", collect, []).option("--tags <csv>", "comma-separated tags").action(
   async (slug, caseName, opts) => {
     try {
       const res = await new ErdoClient().updateEvalCase(slug, caseName, {
         name: caseName,
         input: opts.input,
         setup_messages: opts.setup,
+        target_type: opts.targetType,
+        clear_setup: !!opts.clearSetup,
+        clear_target: !!opts.clearTarget,
         ...caseScoringPayload(opts),
         tags: opts.tags ? opts.tags.split(",").map((t) => t.trim()) : []
       });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@erdoai/cli",
-  "version": "0.55.1",
+  "version": "0.55.3",
   "description": "Erdo CLI — drive datasets, pages, and evals from the terminal or CI",
   "type": "module",
   "bin": {