@erdoai/cli 0.189.0 → 0.193.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +103 -6
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -122,6 +122,35 @@ function parseCSV(text) {
|
|
|
122
122
|
return rows;
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
// src/eval-compare.ts
|
|
126
|
+
function evalResultScored(r) {
|
|
127
|
+
return !r.agent_error && r.failure_classification !== "judge_error";
|
|
128
|
+
}
|
|
129
|
+
function compareEvalRuns(before, after, cases = []) {
|
|
130
|
+
const nameByID = new Map(cases.map((c) => [c.id, c.name]));
|
|
131
|
+
const beforeByCase = new Map(before.map((r) => [r.case_id, r]));
|
|
132
|
+
const afterByCase = new Map(after.map((r) => [r.case_id, r]));
|
|
133
|
+
const ids = Array.from(/* @__PURE__ */ new Set([...beforeByCase.keys(), ...afterByCase.keys()]));
|
|
134
|
+
const rows = ids.map((id) => {
|
|
135
|
+
const b = beforeByCase.get(id);
|
|
136
|
+
const a = afterByCase.get(id);
|
|
137
|
+
const bScore = b && evalResultScored(b) ? b.score : null;
|
|
138
|
+
const aScore = a && evalResultScored(a) ? a.score : null;
|
|
139
|
+
let flip = "";
|
|
140
|
+
if (bScore === null || aScore === null) flip = "unscored";
|
|
141
|
+
else if (!b.passed && a.passed) flip = "now passes";
|
|
142
|
+
else if (b.passed && !a.passed) flip = "now fails";
|
|
143
|
+
return {
|
|
144
|
+
case: nameByID.get(id) ?? id,
|
|
145
|
+
before: bScore,
|
|
146
|
+
after: aScore,
|
|
147
|
+
change: bScore !== null && aScore !== null ? Math.round((aScore - bScore) * 100) / 100 : null,
|
|
148
|
+
flip
|
|
149
|
+
};
|
|
150
|
+
});
|
|
151
|
+
return rows.sort((x, y) => (y.change ?? -Infinity) - (x.change ?? -Infinity));
|
|
152
|
+
}
|
|
153
|
+
|
|
125
154
|
// src/index.ts
|
|
126
155
|
import { password, select } from "@inquirer/prompts";
|
|
127
156
|
import { Argument, Command as Command2, Option as Option2 } from "commander";
|
|
@@ -2867,6 +2896,7 @@ function summariseResults(results, cases = []) {
|
|
|
2867
2896
|
for (const r of results) {
|
|
2868
2897
|
let status;
|
|
2869
2898
|
if (r.agent_error) status = "ERROR";
|
|
2899
|
+
else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
|
|
2870
2900
|
else if (r.passed) status = "PASS";
|
|
2871
2901
|
else status = "FAIL";
|
|
2872
2902
|
const name = nameByID.get(r.case_id) ?? r.case_id;
|
|
@@ -2877,6 +2907,13 @@ function summariseResults(results, cases = []) {
|
|
|
2877
2907
|
}
|
|
2878
2908
|
}
|
|
2879
2909
|
}
|
|
2910
|
+
function printEvalComparison(rows) {
|
|
2911
|
+
const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
|
|
2912
|
+
for (const r of rows) {
|
|
2913
|
+
const change = r.change === null ? "" : `${r.change >= 0 ? "+" : ""}${r.change.toFixed(2)}`;
|
|
2914
|
+
console.log(` ${fmt(r.before)} -> ${fmt(r.after)} ${change.padEnd(6)} ${r.case}${r.flip ? ` (${r.flip})` : ""}`);
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2880
2917
|
function printEvalRunAttribution(run) {
|
|
2881
2918
|
console.log(`source: ${run.triggered_by}`);
|
|
2882
2919
|
if (run.commit_sha) console.log(`commit: ${run.commit_sha}`);
|
|
@@ -4726,14 +4763,36 @@ Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; ag
|
|
|
4726
4763
|
fail(e);
|
|
4727
4764
|
}
|
|
4728
4765
|
});
|
|
4729
|
-
evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").
|
|
4766
|
+
evalCmd.command("results <runId>").description("Show a run's results").option("--json", "print the full JSON").option(
|
|
4767
|
+
"--compare <baselineRunId>",
|
|
4768
|
+
"compare against an earlier run of the same suite: per-case score change and which cases flipped"
|
|
4769
|
+
).action(async (runId, opts) => {
|
|
4730
4770
|
try {
|
|
4731
|
-
const
|
|
4732
|
-
|
|
4771
|
+
const client = new ErdoClient();
|
|
4772
|
+
const data = await client.getEvalRun(runId);
|
|
4773
|
+
if (opts.json && !opts.compare) {
|
|
4733
4774
|
print(data);
|
|
4734
4775
|
return;
|
|
4735
4776
|
}
|
|
4736
4777
|
const { run, results, cases } = data;
|
|
4778
|
+
if (opts.compare) {
|
|
4779
|
+
const baseline = await client.getEvalRun(opts.compare);
|
|
4780
|
+
if (baseline.run.suite_id !== run.suite_id) {
|
|
4781
|
+
throw new Error(
|
|
4782
|
+
`run ${opts.compare} belongs to a different suite; compare two runs of the same suite (erdo eval runs --suite <slug>)`
|
|
4783
|
+
);
|
|
4784
|
+
}
|
|
4785
|
+
const comparison = compareEvalRuns(baseline.results, results, cases.length ? cases : baseline.cases);
|
|
4786
|
+
if (opts.json) {
|
|
4787
|
+
print(comparison);
|
|
4788
|
+
return;
|
|
4789
|
+
}
|
|
4790
|
+
console.log(
|
|
4791
|
+
`avg ${baseline.run.avg_score.toFixed(2)} -> ${run.avg_score.toFixed(2)}, passed ${baseline.run.passed_cases} -> ${run.passed_cases} of ${run.total_cases}`
|
|
4792
|
+
);
|
|
4793
|
+
printEvalComparison(comparison);
|
|
4794
|
+
return;
|
|
4795
|
+
}
|
|
4737
4796
|
console.log(
|
|
4738
4797
|
`${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_score.toFixed(2)}`
|
|
4739
4798
|
);
|
|
@@ -10358,7 +10417,23 @@ knowledgeAssetsCmd.command("list <objectId>").description("List files attached t
|
|
|
10358
10417
|
});
|
|
10359
10418
|
var autoCmd = program.command("automations").description("Scheduled automations (heartbeats)");
|
|
10360
10419
|
function heartbeatBounds(h) {
|
|
10361
|
-
|
|
10420
|
+
const at = h.anchor_time ? ` at:${h.anchor_time}${h.timezone ? ` ${h.timezone}` : ""}` : "";
|
|
10421
|
+
const after = h.run_after_id ? ` after:${h.run_after_id}` : "";
|
|
10422
|
+
return `${at}${after}${h.run_at ? ` run_at:${h.run_at}` : ""}${h.ends_at ? ` ends_at:${h.ends_at}` : ""}`;
|
|
10423
|
+
}
|
|
10424
|
+
function clockTime(flag, v) {
|
|
10425
|
+
if (v === void 0) return void 0;
|
|
10426
|
+
const value = v.trim();
|
|
10427
|
+
if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(value)) {
|
|
10428
|
+
throw new Error(`${flag} must be a 24-hour clock time such as 21:02 \u2014 got ${JSON.stringify(v)}`);
|
|
10429
|
+
}
|
|
10430
|
+
return value;
|
|
10431
|
+
}
|
|
10432
|
+
function afterWhenFlag(v) {
|
|
10433
|
+
if (v === void 0) return void 0;
|
|
10434
|
+
if (v === "success") return true;
|
|
10435
|
+
if (v === "finished") return false;
|
|
10436
|
+
throw new Error(`--after-when must be "success" or "finished" \u2014 got ${JSON.stringify(v)}`);
|
|
10362
10437
|
}
|
|
10363
10438
|
function scheduleBound(flag, v) {
|
|
10364
10439
|
if (v === void 0) return void 0;
|
|
@@ -10370,7 +10445,7 @@ function scheduleBound(flag, v) {
|
|
|
10370
10445
|
}
|
|
10371
10446
|
autoCmd.command("create <name>").description(
|
|
10372
10447
|
"Create an automation. Pass --run-at for one-off work at a date (it runs once, then stops for good) instead of a recurring automation that checks the date"
|
|
10373
|
-
).option("--description <text>", "What the automation does").option("--instructions <text>", "Instructions for each run (agent automation)").option("--script-js <js>", "Deterministic script body (scripted automation, no LLM)").option("--script-file <path>", "Read the script body from a file (scripted automation)").requiredOption("--interval <minutes>", "Interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "Scheduling timezone, e.g. America/New_York (default UTC)").option("--run-at <rfc3339>", "Run once at or after this instant, then stop permanently").option("--ends-at <rfc3339>", "Never run after this instant").addOption(new Option2("--agent-key <key>", "Executor for an agent automation").choices(["erdo.heartbeat-executor", "erdo.workflow-executor"])).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").action(
|
|
10448
|
+
).option("--description <text>", "What the automation does").option("--instructions <text>", "Instructions for each run (agent automation)").option("--script-js <js>", "Deterministic script body (scripted automation, no LLM)").option("--script-file <path>", "Read the script body from a file (scripted automation)").requiredOption("--interval <minutes>", "Interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "Scheduling timezone, e.g. America/New_York (default UTC)").option("--at <HH:MM>", "Clock time of each run in --timezone, e.g. 21:02").option("--run-after <id>", "Also start when this automation finishes").option("--after-when <when>", "success or finished (default finished)").option("--after-delay <minutes>", "Wait this many minutes after the predecessor finishes", (v) => parseInt(v, 10)).option("--instead-of-schedule", "Run only after --run-after, with no clock of its own").option("--run-at <rfc3339>", "Run once at or after this instant, then stop permanently").option("--ends-at <rfc3339>", "Never run after this instant").addOption(new Option2("--agent-key <key>", "Executor for an agent automation").choices(["erdo.heartbeat-executor", "erdo.workflow-executor"])).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").action(
|
|
10374
10449
|
async (name, opts) => {
|
|
10375
10450
|
try {
|
|
10376
10451
|
if (opts.scriptJs && opts.scriptFile) {
|
|
@@ -10385,6 +10460,11 @@ autoCmd.command("create <name>").description(
|
|
|
10385
10460
|
script_js: scriptJs,
|
|
10386
10461
|
interval_minutes: opts.interval,
|
|
10387
10462
|
timezone: opts.timezone,
|
|
10463
|
+
anchor_time: clockTime("--at", opts.at),
|
|
10464
|
+
run_after_id: opts.runAfter,
|
|
10465
|
+
run_after_on_success: afterWhenFlag(opts.afterWhen),
|
|
10466
|
+
run_after_delay_minutes: opts.afterDelay,
|
|
10467
|
+
run_after_replaces_schedule: opts.insteadOfSchedule ? true : void 0,
|
|
10388
10468
|
run_at: scheduleBound("--run-at", opts.runAt),
|
|
10389
10469
|
ends_at: scheduleBound("--ends-at", opts.endsAt),
|
|
10390
10470
|
agent_key: opts.agentKey,
|
|
@@ -10427,7 +10507,7 @@ autoCmd.command("run <id>").description("Trigger an automation now").action(asyn
|
|
|
10427
10507
|
autoCmd.command("update <id>").description("Edit an automation \u2014 rename, reschedule, or replace its script/instructions").option("--name <name>", "New name").option("--description <text>", "New description").option("--instructions <text>", "Replacement instructions (agent automation)").addOption(new Option2("--agent-key <key>", "Executor for an agent automation; omit to keep current").choices(["erdo.heartbeat-executor", "erdo.workflow-executor"])).option("--script-js <js>", "Replacement script body (scripted automation)").option(
|
|
10428
10508
|
"--script-file <path>",
|
|
10429
10509
|
"Read the replacement script body from a file (scripted automation)"
|
|
10430
|
-
).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").option("--interval <minutes>", "New interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "New scheduling timezone, e.g. America/New_York").option("--run-at <rfc3339>", "New one-off fire time: runs once at or after it, then stops permanently (omit to keep)").option("--ends-at <rfc3339>", "New end time after which it never runs (omit to keep)").action(
|
|
10510
|
+
).addOption(new Option2("--conversation <target>", "Target 'new' (fresh thread) or 'direct' conversation").choices(["new", "direct"])).option("--thread-id <id>", "Post each run into an existing conversation UUID").option("--persona-id <id>", "Org-enabled persona UUID to speak as").option("--interval <minutes>", "New interval in minutes (minimum 5)", (v) => parseInt(v, 10)).option("--timezone <tz>", "New scheduling timezone, e.g. America/New_York").option("--at <HH:MM>", "Clock time of each run in the automation timezone, e.g. 21:02").option("--clear-at", "Clear the clock time so the next run is one interval after this save").option("--run-after <id>", "Start when this automation finishes").option("--clear-run-after", "Stop following another automation").option("--after-when <when>", "success or finished").option("--after-delay <minutes>", "Wait this many minutes after the predecessor finishes", (v) => parseInt(v, 10)).option("--instead-of-schedule", "Run only from the chain, with no clock").option("--keep-schedule", "Keep the clock as well as the chain").option("--run-at <rfc3339>", "New one-off fire time: runs once at or after it, then stops permanently (omit to keep)").option("--ends-at <rfc3339>", "New end time after which it never runs (omit to keep)").action(
|
|
10431
10511
|
async (id, opts) => {
|
|
10432
10512
|
try {
|
|
10433
10513
|
if (opts.scriptJs && opts.scriptFile) {
|
|
@@ -10435,6 +10515,18 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
|
|
|
10435
10515
|
return;
|
|
10436
10516
|
}
|
|
10437
10517
|
const scriptJs = opts.scriptFile ? readFileSync4(opts.scriptFile, "utf8") : opts.scriptJs;
|
|
10518
|
+
if (opts.at !== void 0 && opts.clearAt) {
|
|
10519
|
+
fail(new Error("pass either --at or --clear-at, not both"));
|
|
10520
|
+
return;
|
|
10521
|
+
}
|
|
10522
|
+
if (opts.runAfter && opts.clearRunAfter) {
|
|
10523
|
+
fail(new Error("pass either --run-after or --clear-run-after, not both"));
|
|
10524
|
+
return;
|
|
10525
|
+
}
|
|
10526
|
+
if (opts.insteadOfSchedule && opts.keepSchedule) {
|
|
10527
|
+
fail(new Error("pass either --instead-of-schedule or --keep-schedule, not both"));
|
|
10528
|
+
return;
|
|
10529
|
+
}
|
|
10438
10530
|
const body = {
|
|
10439
10531
|
name: opts.name,
|
|
10440
10532
|
description: opts.description,
|
|
@@ -10443,6 +10535,11 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
|
|
|
10443
10535
|
script_js: scriptJs,
|
|
10444
10536
|
interval_minutes: opts.interval,
|
|
10445
10537
|
timezone: opts.timezone,
|
|
10538
|
+
anchor_time: opts.clearAt ? "" : clockTime("--at", opts.at),
|
|
10539
|
+
run_after_id: opts.clearRunAfter ? "" : opts.runAfter,
|
|
10540
|
+
run_after_on_success: afterWhenFlag(opts.afterWhen),
|
|
10541
|
+
run_after_delay_minutes: opts.afterDelay,
|
|
10542
|
+
run_after_replaces_schedule: opts.insteadOfSchedule ? true : opts.keepSchedule ? false : void 0,
|
|
10446
10543
|
run_at: scheduleBound("--run-at", opts.runAt),
|
|
10447
10544
|
ends_at: scheduleBound("--ends-at", opts.endsAt),
|
|
10448
10545
|
conversation: opts.conversation,
|