@erdoai/cli 0.189.0 → 0.193.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +103 -6
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -122,6 +122,35 @@ function parseCSV(text) {
122
122
  return rows;
123
123
  }
124
124
 
125
+ // src/eval-compare.ts
126
+ function evalResultScored(r) {
127
+ return !r.agent_error && r.failure_classification !== "judge_error";
128
+ }
129
+ function compareEvalRuns(before, after, cases = []) {
130
+ const nameByID = new Map(cases.map((c) => [c.id, c.name]));
131
+ const beforeByCase = new Map(before.map((r) => [r.case_id, r]));
132
+ const afterByCase = new Map(after.map((r) => [r.case_id, r]));
133
+ const ids = Array.from(/* @__PURE__ */ new Set([...beforeByCase.keys(), ...afterByCase.keys()]));
134
+ const rows = ids.map((id) => {
135
+ const b = beforeByCase.get(id);
136
+ const a = afterByCase.get(id);
137
+ const bScore = b && evalResultScored(b) ? b.score : null;
138
+ const aScore = a && evalResultScored(a) ? a.score : null;
139
+ let flip = "";
140
+ if (bScore === null || aScore === null) flip = "unscored";
141
+ else if (!b.passed && a.passed) flip = "now passes";
142
+ else if (b.passed && !a.passed) flip = "now fails";
143
+ return {
144
+ case: nameByID.get(id) ?? id,
145
+ before: bScore,
146
+ after: aScore,
147
+ change: bScore !== null && aScore !== null ? Math.round((aScore - bScore) * 100) / 100 : null,
148
+ flip
149
+ };
150
+ });
151
+ return rows.sort((x, y) => (y.change ?? -Infinity) - (x.change ?? -Infinity));
152
+ }
153
+
125
154
  // src/index.ts
126
155
  import { password, select } from "@inquirer/prompts";
127
156
  import { Argument, Command as Command2, Option as Option2 } from "commander";
@@ -2867,6 +2896,7 @@ function summariseResults(results, cases = []) {
2867
2896
  for (const r of results) {
2868
2897
  let status;
2869
2898
  if (r.agent_error) status = "ERROR";
2899
+ else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
2870
2900
  else if (r.passed) status = "PASS";
2871
2901
  else status = "FAIL";
2872
2902
  const name = nameByID.get(r.case_id) ?? r.case_id;
@@ -2877,6 +2907,13 @@ function summariseResults(results, cases = []) {
2877
2907
  }
2878
2908
  }
2879
2909
  }
2910
+ function printEvalComparison(rows) {
2911
+ const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
2912
+ for (const r of rows) {
2913
+ const change = r.change === null ? "" : `${r.change >= 0 ? "+" : ""}${r.change.toFixed(2)}`;
2914
+ console.log(` ${fmt(r.before)} -> ${fmt(r.after)} ${change.padEnd(6)} ${r.case}${r.flip ? ` (${r.flip})` : ""}`);
2915
+ }
2916
+ }
2880
2917
  function printEvalRunAttribution(run) {
2881
2918
  console.log(`source: ${run.triggered_by}`);
2882
2919
  if (run.commit_sha) console.log(`commit: ${run.commit_sha}`);
@@ -4726,14 +4763,36 @@ Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; ag
4726
4763
  fail(e);
4727
4764
  }
4728
4765
  });
4729
- evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").action(async (runId, opts) => {
4766
+ evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").option(
4767
+ "--compare <baselineRunId>",
4768
+ "compare against an earlier run of the same suite: per-case score change and which cases flipped"
4769
+ ).action(async (runId, opts) => {
4730
4770
  try {
4731
- const data = await new ErdoClient().getEvalRun(runId);
4732
- if (opts.json) {
4771
+ const client = new ErdoClient();
4772
+ const data = await client.getEvalRun(runId);
4773
+ if (opts.json && !opts.compare) {
4733
4774
  print(data);
4734
4775
  return;
4735
4776
  }
4736
4777
  const { run, results, cases } = data;
4778
+ if (opts.compare) {
4779
+ const baseline = await client.getEvalRun(opts.compare);
4780
+ if (baseline.run.suite_id !== run.suite_id) {
4781
+ throw new Error(
4782
+ `run ${opts.compare} belongs to a different suite; compare two runs of the same suite (erdo eval runs --suite <slug>)`
4783
+ );
4784
+ }
4785
+ const comparison = compareEvalRuns(baseline.results, results, cases.length ? cases : baseline.cases);
4786
+ if (opts.json) {
4787
+ print(comparison);
4788
+ return;
4789
+ }
4790
+ console.log(
4791
+ `avg ${baseline.run.avg_score.toFixed(2)} -> ${run.avg_score.toFixed(2)}, passed ${baseline.run.passed_cases} -> ${run.passed_cases} of ${run.total_cases}`
4792
+ );
4793
+ printEvalComparison(comparison);
4794
+ return;
4795
+ }
4737
4796
  console.log(
4738
4797
  `${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_score.toFixed(2)}`
4739
4798
  );
@@ -10358,7 +10417,23 @@ knowledgeAssetsCmd.command("list <objectId>").description("List files attached t
10358
10417
  });
10359
10418
  var autoCmd = program.command("automations").description("Scheduled automations (heartbeats)");
10360
10419
  function heartbeatBounds(h) {
10361
- return `${h.run_at ? ` run_at:${h.run_at}` : ""}${h.ends_at ? ` ends_at:${h.ends_at}` : ""}`;
10420
+ const at = h.anchor_time ? ` at:${h.anchor_time}${h.timezone ? ` ${h.timezone}` : ""}` : "";
10421
+ const after = h.run_after_id ? ` after:${h.run_after_id}` : "";
10422
+ return `${at}${after}${h.run_at ? ` run_at:${h.run_at}` : ""}${h.ends_at ? ` ends_at:${h.ends_at}` : ""}`;
10423
+ }
10424
+ function clockTime(flag, v) {
10425
+ if (v === void 0) return void 0;
10426
+ const value = v.trim();
10427
+ if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(value)) {
10428
+ throw new Error(`${flag} must be a 24-hour clock time such as 21:02 \u2014 got ${JSON.stringify(v)}`);
10429
+ }
10430
+ return value;
10431
+ }
10432
+ function afterWhenFlag(v) {
10433
+ if (v === void 0) return void 0;
10434
+ if (v === "success") return true;
10435
+ if (v === "finished") return false;
10436
+ throw new Error(`--after-when must be "success" or "finished" \u2014 got ${JSON.stringify(v)}`);
10362
10437
  }
10363
10438
  function scheduleBound(flag, v) {
10364
10439
  if (v === void 0) return void 0;
@@ -10370,7 +10445,7 @@ function scheduleBound(flag, v) {
10370
10445
  }
10371
10446
  autoCmd.command("create <name>").description(
10372
10447
  "Create an automation. Pass --run-at for one-off work at a date (it runs once, then stops for good) instead of a recurring automation that checks the date"
10373
- ).option("--description <text>", "What the automation does").option("--instructions <text>", "Instructions for each run (agent automation)").option("--script-js <js>", "Deterministic script body (scripted automation, no LLM)").option("--script-file <path>", "Read the script body from a file (scripted automation)").requiredOption("--interval <minutes>", "Interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "Scheduling timezone, e.g. America/New_York (default UTC)").option("--run-at <rfc3339>", "Run once at or after this instant, then stop permanently").option("--ends-at <rfc3339>", "Never run after this instant").addOption(new Option2("--agent-key <key>", "Executor for an agent automation").choices(["erdo.heartbeat-executor", "erdo.workflow-executor"])).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").action(
10448
+ ).option("--description <text>", "What the automation does").option("--instructions <text>", "Instructions for each run (agent automation)").option("--script-js <js>", "Deterministic script body (scripted automation, no LLM)").option("--script-file <path>", "Read the script body from a file (scripted automation)").requiredOption("--interval <minutes>", "Interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "Scheduling timezone, e.g. America/New_York (default UTC)").option("--at <HH:MM>", "Clock time of each run in --timezone, e.g. 21:02").option("--run-after <id>", "Also start when this automation finishes").option("--after-when <when>", "success or finished (default finished)").option("--after-delay <minutes>", "Wait this many minutes after the predecessor finishes", (v) => parseInt(v, 10)).option("--instead-of-schedule", "Run only after --run-after, with no clock of its own").option("--run-at <rfc3339>", "Run once at or after this instant, then stop permanently").option("--ends-at <rfc3339>", "Never run after this instant").addOption(new Option2("--agent-key <key>", "Executor for an agent automation").choices(["erdo.heartbeat-executor", "erdo.workflow-executor"])).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").action(
10374
10449
  async (name, opts) => {
10375
10450
  try {
10376
10451
  if (opts.scriptJs && opts.scriptFile) {
@@ -10385,6 +10460,11 @@ autoCmd.command("create <name>").description(
10385
10460
  script_js: scriptJs,
10386
10461
  interval_minutes: opts.interval,
10387
10462
  timezone: opts.timezone,
10463
+ anchor_time: clockTime("--at", opts.at),
10464
+ run_after_id: opts.runAfter,
10465
+ run_after_on_success: afterWhenFlag(opts.afterWhen),
10466
+ run_after_delay_minutes: opts.afterDelay,
10467
+ run_after_replaces_schedule: opts.insteadOfSchedule ? true : void 0,
10388
10468
  run_at: scheduleBound("--run-at", opts.runAt),
10389
10469
  ends_at: scheduleBound("--ends-at", opts.endsAt),
10390
10470
  agent_key: opts.agentKey,
@@ -10427,7 +10507,7 @@ autoCmd.command("run <id>").description("Trigger an automation now").action(asyn
10427
10507
  autoCmd.command("update <id>").description("Edit an automation \u2014 rename, reschedule, or replace its script/instructions").option("--name <name>", "New name").option("--description <text>", "New description").option("--instructions <text>", "Replacement instructions (agent automation)").addOption(new Option2("--agent-key <key>", "Executor for an agent automation; omit to keep current").choices(["erdo.heartbeat-executor", "erdo.workflow-executor"])).option("--script-js <js>", "Replacement script body (scripted automation)").option(
10428
10508
  "--script-file <path>",
10429
10509
  "Read the replacement script body from a file (scripted automation)"
10430
- ).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").option("--interval <minutes>", "New interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "New scheduling timezone, e.g. America/New_York").option("--run-at <rfc3339>", "New one-off fire time: runs once at or after it, then stops permanently (omit to keep)").option("--ends-at <rfc3339>", "New end time after which it never runs (omit to keep)").action(
10510
+ ).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").option("--interval <minutes>", "New interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "New scheduling timezone, e.g. America/New_York").option("--at <HH:MM>", "Clock time of each run in the automation timezone, e.g. 21:02").option("--clear-at", "Clear the clock time so the next run is one interval after this save").option("--run-after <id>", "Start when this automation finishes").option("--clear-run-after", "Stop following another automation").option("--after-when <when>", "success or finished").option("--after-delay <minutes>", "Wait this many minutes after the predecessor finishes", (v) => parseInt(v, 10)).option("--instead-of-schedule", "Run only from the chain, with no clock").option("--keep-schedule", "Keep the clock as well as the chain").option("--run-at <rfc3339>", "New one-off fire time: runs once at or after it, then stops permanently (omit to keep)").option("--ends-at <rfc3339>", "New end time after which it never runs (omit to keep)").action(
10431
10511
  async (id, opts) => {
10432
10512
  try {
10433
10513
  if (opts.scriptJs && opts.scriptFile) {
@@ -10435,6 +10515,18 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
10435
10515
  return;
10436
10516
  }
10437
10517
  const scriptJs = opts.scriptFile ? readFileSync4(opts.scriptFile, "utf8") : opts.scriptJs;
10518
+ if (opts.at !== void 0 && opts.clearAt) {
10519
+ fail(new Error("pass either --at or --clear-at, not both"));
10520
+ return;
10521
+ }
10522
+ if (opts.runAfter && opts.clearRunAfter) {
10523
+ fail(new Error("pass either --run-after or --clear-run-after, not both"));
10524
+ return;
10525
+ }
10526
+ if (opts.insteadOfSchedule && opts.keepSchedule) {
10527
+ fail(new Error("pass either --instead-of-schedule or --keep-schedule, not both"));
10528
+ return;
10529
+ }
10438
10530
  const body = {
10439
10531
  name: opts.name,
10440
10532
  description: opts.description,
@@ -10443,6 +10535,11 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
10443
10535
  script_js: scriptJs,
10444
10536
  interval_minutes: opts.interval,
10445
10537
  timezone: opts.timezone,
10538
+ anchor_time: opts.clearAt ? "" : clockTime("--at", opts.at),
10539
+ run_after_id: opts.clearRunAfter ? "" : opts.runAfter,
10540
+ run_after_on_success: afterWhenFlag(opts.afterWhen),
10541
+ run_after_delay_minutes: opts.afterDelay,
10542
+ run_after_replaces_schedule: opts.insteadOfSchedule ? true : opts.keepSchedule ? false : void 0,
10446
10543
  run_at: scheduleBound("--run-at", opts.runAt),
10447
10544
  ends_at: scheduleBound("--ends-at", opts.endsAt),
10448
10545
  conversation: opts.conversation,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.189.0",
3
+ "version": "0.193.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {