@erdoai/cli 0.201.0 → 0.202.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +58 -8
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -150,7 +150,7 @@ function parseCSV(text) {
|
|
|
150
150
|
|
|
151
151
|
// src/eval-compare.ts
|
|
152
152
|
function evalResultScored(r) {
|
|
153
|
-
return !r.agent_error && r.failure_classification !== "judge_error";
|
|
153
|
+
return !r.agent_error && r.failure_classification !== "judge_error" && r.scoring_status !== "awaiting_truth";
|
|
154
154
|
}
|
|
155
155
|
function compareEvalRuns(before, after, cases = []) {
|
|
156
156
|
const nameByID = new Map(cases.map((c) => [c.id, c.name]));
|
|
@@ -1131,8 +1131,19 @@ var ErdoClient = class {
|
|
|
1131
1131
|
}
|
|
1132
1132
|
);
|
|
1133
1133
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1134
|
+
scoreEvalClaims(slug, input) {
|
|
1135
|
+
return this.request(
|
|
1136
|
+
"POST",
|
|
1137
|
+
`/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
|
|
1138
|
+
input
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
getEvalLeaderboard(slug, days, truthVersion) {
|
|
1142
|
+
const params = new URLSearchParams();
|
|
1143
|
+
if (days && days > 0) params.set("days", String(days));
|
|
1144
|
+
if (truthVersion && truthVersion > 0) params.set("truth_version", String(truthVersion));
|
|
1145
|
+
const qs = params.toString();
|
|
1146
|
+
const q = qs ? `?${qs}` : "";
|
|
1136
1147
|
return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
|
|
1137
1148
|
}
|
|
1138
1149
|
getEvalRun(runID) {
|
|
@@ -3031,11 +3042,13 @@ function summariseResults(results, cases = []) {
|
|
|
3031
3042
|
for (const r of results) {
|
|
3032
3043
|
let status;
|
|
3033
3044
|
if (r.agent_error) status = "ERROR";
|
|
3045
|
+
else if (r.scoring_status === "awaiting_truth") status = "AWAITING-TRUTH";
|
|
3034
3046
|
else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
|
|
3035
3047
|
else if (r.passed) status = "PASS";
|
|
3036
3048
|
else status = "FAIL";
|
|
3037
3049
|
const name = nameByID.get(r.case_id) ?? r.case_id;
|
|
3038
|
-
|
|
3050
|
+
const scoreText = r.scoring_status === "awaiting_truth" ? " -- " : r.score.toFixed(2);
|
|
3051
|
+
console.log(` [${status}] ${scoreText} ${name}`);
|
|
3039
3052
|
if (r.candidate_cost_usd) console.log(` candidate cost: $${r.candidate_cost_usd.toFixed(4)} (reported by the candidate, not judge cost)`);
|
|
3040
3053
|
if (r.agent_error) console.log(` agent error: ${r.agent_error}`);
|
|
3041
3054
|
for (const c of r.judge_criteria_scores || []) {
|
|
@@ -3043,6 +3056,22 @@ function summariseResults(results, cases = []) {
|
|
|
3043
3056
|
}
|
|
3044
3057
|
}
|
|
3045
3058
|
}
|
|
3059
|
+
function formatCriterionAverages(averages) {
|
|
3060
|
+
const withCriteria = (averages ?? []).filter((m) => m.criteria.length > 0);
|
|
3061
|
+
if (withCriteria.length === 0) return [];
|
|
3062
|
+
const lines = ["\nCriterion averages (avg score (n) per model)"];
|
|
3063
|
+
for (const m of withCriteria) {
|
|
3064
|
+
lines.push(` ${m.model}`);
|
|
3065
|
+
for (const c of m.criteria) lines.push(` ${c.avg_score.toFixed(2)} (${c.n}) ${c.criterion}`);
|
|
3066
|
+
}
|
|
3067
|
+
return lines;
|
|
3068
|
+
}
|
|
3069
|
+
function parseEvalTruthVersion(v) {
|
|
3070
|
+
const t = v.trim().toLowerCase();
|
|
3071
|
+
if (t === "new") return "new";
|
|
3072
|
+
if (/^[1-9][0-9]*$/.test(t)) return t;
|
|
3073
|
+
throw new Error(`--truth-version must be "new" or a positive integer, got "${v}"`);
|
|
3074
|
+
}
|
|
3046
3075
|
function printEvalComparison(rows) {
|
|
3047
3076
|
const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
|
|
3048
3077
|
for (const r of rows) {
|
|
@@ -4779,7 +4808,7 @@ function parseStrictBoolean(value) {
|
|
|
4779
4808
|
throw new Error("value must be true or false");
|
|
4780
4809
|
}
|
|
4781
4810
|
}
|
|
4782
|
-
evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
|
|
4811
|
+
evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (empty string resets to default)").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
|
|
4783
4812
|
async (slug, opts) => {
|
|
4784
4813
|
try {
|
|
4785
4814
|
const res = await new ErdoClient().updateEvalSuite(slug, {
|
|
@@ -4787,6 +4816,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
|
|
|
4787
4816
|
description: opts.description,
|
|
4788
4817
|
agent_key: opts.agent,
|
|
4789
4818
|
judge_model: opts.judge,
|
|
4819
|
+
claims_verifier_model: opts.claimsVerifierModel,
|
|
4790
4820
|
pass_threshold: opts.threshold,
|
|
4791
4821
|
evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
|
|
4792
4822
|
cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
|
|
@@ -4797,7 +4827,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
|
|
|
4797
4827
|
}
|
|
4798
4828
|
}
|
|
4799
4829
|
);
|
|
4800
|
-
evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
|
|
4830
|
+
evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (default if omitted)").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
|
|
4801
4831
|
acc.push(v);
|
|
4802
4832
|
return acc;
|
|
4803
4833
|
}, []).action(
|
|
@@ -4809,6 +4839,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
|
|
|
4809
4839
|
description: opts.description,
|
|
4810
4840
|
agent_key: opts.agent,
|
|
4811
4841
|
judge_model: opts.judge,
|
|
4842
|
+
claims_verifier_model: opts.claimsVerifierModel,
|
|
4812
4843
|
pass_threshold: opts.threshold,
|
|
4813
4844
|
evaluate_artifact: !!opts.evaluateArtifact,
|
|
4814
4845
|
cron_enabled: opts.cron,
|
|
@@ -4899,9 +4930,27 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
|
|
|
4899
4930
|
fail(e);
|
|
4900
4931
|
}
|
|
4901
4932
|
});
|
|
4902
|
-
evalCmd.command("
|
|
4933
|
+
evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
|
|
4934
|
+
try {
|
|
4935
|
+
const truthVersion = parseEvalTruthVersion(opts.truthVersion);
|
|
4936
|
+
const res = await new ErdoClient().scoreEvalClaims(suite, {
|
|
4937
|
+
truth_version: truthVersion,
|
|
4938
|
+
regenerate_criteria: !!opts.regenerateCriteria
|
|
4939
|
+
});
|
|
4940
|
+
console.log(`pass: ${res.pass_id}`);
|
|
4941
|
+
console.log(
|
|
4942
|
+
`Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
|
|
4943
|
+
);
|
|
4944
|
+
} catch (e) {
|
|
4945
|
+
fail(e);
|
|
4946
|
+
}
|
|
4947
|
+
});
|
|
4948
|
+
evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
|
|
4949
|
+
if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
|
|
4950
|
+
return parseInt(v, 10);
|
|
4951
|
+
}).action(async (slug, opts) => {
|
|
4903
4952
|
try {
|
|
4904
|
-
const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
|
|
4953
|
+
const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days, opts.truthVersion);
|
|
4905
4954
|
if (res.models.length === 0) {
|
|
4906
4955
|
console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
|
|
4907
4956
|
return;
|
|
@@ -4933,6 +4982,7 @@ ${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`)
|
|
|
4933
4982
|
}
|
|
4934
4983
|
console.log(line);
|
|
4935
4984
|
}
|
|
4985
|
+
for (const line of formatCriterionAverages(res.criterion_averages)) console.log(line);
|
|
4936
4986
|
console.log(
|
|
4937
4987
|
`
|
|
4938
4988
|
Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
|