@erdoai/cli 0.213.0 → 0.214.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +24 -5
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1197,11 +1197,10 @@ var ErdoClient = class {
|
|
|
1197
1197
|
);
|
|
1198
1198
|
}
|
|
1199
1199
|
scoreEvalClaims(slug, input) {
|
|
1200
|
-
return this.request(
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
);
|
|
1200
|
+
return this.request("POST", `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`, input);
|
|
1201
|
+
}
|
|
1202
|
+
getEvalClaimsPass(passId) {
|
|
1203
|
+
return this.request("GET", `/v1/evals/claims-passes/${encodeURIComponent(passId)}`);
|
|
1205
1204
|
}
|
|
1206
1205
|
getEvalLeaderboard(slug, days, truthVersion) {
|
|
1207
1206
|
const params = new URLSearchParams();
|
|
@@ -5025,6 +5024,9 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
|
|
|
5025
5024
|
fail(e);
|
|
5026
5025
|
}
|
|
5027
5026
|
});
|
|
5027
|
+
function usdFromMillicents(m) {
|
|
5028
|
+
return `$${(m / 1e5).toFixed(2)}`;
|
|
5029
|
+
}
|
|
5028
5030
|
evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against: new starts a new version, or continues the latest when a pass left it unfinished (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
|
|
5029
5031
|
try {
|
|
5030
5032
|
const truthVersion = parseEvalTruthVersion(opts.truthVersion);
|
|
@@ -5033,6 +5035,10 @@ evalCmd.command("score <suite>").description("Score awaiting claims answers agai
|
|
|
5033
5035
|
regenerate_criteria: !!opts.regenerateCriteria
|
|
5034
5036
|
});
|
|
5035
5037
|
console.log(`pass: ${res.pass_id}`);
|
|
5038
|
+
console.log(
|
|
5039
|
+
`spend: stops at ${usdFromMillicents(res.pass_ceiling_millicents)}; this suite's passes spent ${usdFromMillicents(res.suite_spent_last_day_millicents)} of ${usdFromMillicents(res.suite_daily_ceiling_millicents)} in the last 24h`
|
|
5040
|
+
);
|
|
5041
|
+
console.log(`Watch it with: erdo eval pass ${res.pass_id}`);
|
|
5036
5042
|
console.log(
|
|
5037
5043
|
`Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
|
|
5038
5044
|
);
|
|
@@ -5040,6 +5046,19 @@ evalCmd.command("score <suite>").description("Score awaiting claims answers agai
|
|
|
5040
5046
|
fail(e);
|
|
5041
5047
|
}
|
|
5042
5048
|
});
|
|
5049
|
+
evalCmd.command("pass <passId>").description("Show a claims pass's status and LLM spend so far (staff only)").action(async (passId) => {
|
|
5050
|
+
try {
|
|
5051
|
+
const p = await new ErdoClient().getEvalClaimsPass(passId);
|
|
5052
|
+
console.log(`pass: ${p.pass_id}`);
|
|
5053
|
+
console.log(`status: ${p.status}${p.resume_attempts > 0 ? ` (resumed ${p.resume_attempts}x)` : ""}`);
|
|
5054
|
+
console.log(`spend: ${usdFromMillicents(p.cost_millicents)} of ${usdFromMillicents(p.ceiling_millicents)} ceiling`);
|
|
5055
|
+
console.log(`scored: ${p.cases_scored} case(s), ${p.results_scored} answer(s)`);
|
|
5056
|
+
console.log(`started: ${p.started_at}${p.completed_at ? `, finished: ${p.completed_at}` : ""}`);
|
|
5057
|
+
if (p.error) console.log(`note: ${p.error}`);
|
|
5058
|
+
} catch (e) {
|
|
5059
|
+
fail(e);
|
|
5060
|
+
}
|
|
5061
|
+
});
|
|
5043
5062
|
evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
|
|
5044
5063
|
if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
|
|
5045
5064
|
return parseInt(v, 10);
|