@erdoai/cli 0.213.0 → 0.214.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +24 -5
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -1197,11 +1197,10 @@ var ErdoClient = class {
1197
1197
  );
1198
1198
  }
1199
1199
  scoreEvalClaims(slug, input) {
1200
- return this.request(
1201
- "POST",
1202
- `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
1203
- input
1204
- );
1200
+ return this.request("POST", `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`, input);
1201
+ }
1202
+ getEvalClaimsPass(passId) {
1203
+ return this.request("GET", `/v1/evals/claims-passes/${encodeURIComponent(passId)}`);
1205
1204
  }
1206
1205
  getEvalLeaderboard(slug, days, truthVersion) {
1207
1206
  const params = new URLSearchParams();
@@ -5025,6 +5024,9 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
5025
5024
  fail(e);
5026
5025
  }
5027
5026
  });
5027
+ function usdFromMillicents(m) {
5028
+ return `$${(m / 1e5).toFixed(2)}`;
5029
+ }
5028
5030
  evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against: new starts a new version, or continues the latest when a pass left it unfinished (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
5029
5031
  try {
5030
5032
  const truthVersion = parseEvalTruthVersion(opts.truthVersion);
@@ -5033,6 +5035,10 @@ evalCmd.command("score <suite>").description("Score awaiting claims answers agai
5033
5035
  regenerate_criteria: !!opts.regenerateCriteria
5034
5036
  });
5035
5037
  console.log(`pass: ${res.pass_id}`);
5038
+ console.log(
5039
+ `spend: stops at ${usdFromMillicents(res.pass_ceiling_millicents)}; this suite's passes spent ${usdFromMillicents(res.suite_spent_last_day_millicents)} of ${usdFromMillicents(res.suite_daily_ceiling_millicents)} in the last 24h`
5040
+ );
5041
+ console.log(`Watch it with: erdo eval pass ${res.pass_id}`);
5036
5042
  console.log(
5037
5043
  `Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
5038
5044
  );
@@ -5040,6 +5046,19 @@ evalCmd.command("score <suite>").description("Score awaiting claims answers agai
5040
5046
  fail(e);
5041
5047
  }
5042
5048
  });
5049
+ evalCmd.command("pass <passId>").description("Show a claims pass's status and LLM spend so far (staff only)").action(async (passId) => {
5050
+ try {
5051
+ const p = await new ErdoClient().getEvalClaimsPass(passId);
5052
+ console.log(`pass: ${p.pass_id}`);
5053
+ console.log(`status: ${p.status}${p.resume_attempts > 0 ? ` (resumed ${p.resume_attempts}x)` : ""}`);
5054
+ console.log(`spend: ${usdFromMillicents(p.cost_millicents)} of ${usdFromMillicents(p.ceiling_millicents)} ceiling`);
5055
+ console.log(`scored: ${p.cases_scored} case(s), ${p.results_scored} answer(s)`);
5056
+ console.log(`started: ${p.started_at}${p.completed_at ? `, finished: ${p.completed_at}` : ""}`);
5057
+ if (p.error) console.log(`note: ${p.error}`);
5058
+ } catch (e) {
5059
+ fail(e);
5060
+ }
5061
+ });
5043
5062
  evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
5044
5063
  if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
5045
5064
  return parseInt(v, 10);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.213.0",
3
+ "version": "0.214.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {