@erdoai/cli 0.212.0 → 0.214.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +41 -10
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -1197,11 +1197,10 @@ var ErdoClient = class {
1197
1197
  );
1198
1198
  }
1199
1199
  scoreEvalClaims(slug, input) {
1200
- return this.request(
1201
- "POST",
1202
- `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
1203
- input
1204
- );
1200
+ return this.request("POST", `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`, input);
1201
+ }
1202
+ getEvalClaimsPass(passId) {
1203
+ return this.request("GET", `/v1/evals/claims-passes/${encodeURIComponent(passId)}`);
1205
1204
  }
1206
1205
  getEvalLeaderboard(slug, days, truthVersion) {
1207
1206
  const params = new URLSearchParams();
@@ -2302,7 +2301,12 @@ var ErdoClient = class {
2302
2301
  }
2303
2302
  // --- automations (heartbeats) ---
2304
2303
  listHeartbeats(opts = {}) {
2305
- return this.request("GET", opts.includeRefresh ? "/v1/heartbeats?include_refresh=true" : "/v1/heartbeats");
2304
+ const query = new URLSearchParams();
2305
+ if (opts.includeRefresh) query.set("include_refresh", "true");
2306
+ if (opts.limit !== void 0) query.set("limit", String(opts.limit));
2307
+ if (opts.offset !== void 0) query.set("offset", String(opts.offset));
2308
+ const suffix = query.toString() ? `?${query.toString()}` : "";
2309
+ return this.request("GET", `/v1/heartbeats${suffix}`);
2306
2310
  }
2307
2311
  createHeartbeat(body) {
2308
2312
  return this.request("POST", "/v1/heartbeats", body);
@@ -5020,6 +5024,9 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
5020
5024
  fail(e);
5021
5025
  }
5022
5026
  });
5027
+ function usdFromMillicents(m) {
5028
+ return `$${(m / 1e5).toFixed(2)}`;
5029
+ }
5023
5030
  evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against: new starts a new version, or continues the latest when a pass left it unfinished (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
5024
5031
  try {
5025
5032
  const truthVersion = parseEvalTruthVersion(opts.truthVersion);
@@ -5028,6 +5035,10 @@ evalCmd.command("score <suite>").description("Score awaiting claims answers agai
5028
5035
  regenerate_criteria: !!opts.regenerateCriteria
5029
5036
  });
5030
5037
  console.log(`pass: ${res.pass_id}`);
5038
+ console.log(
5039
+ `spend: stops at ${usdFromMillicents(res.pass_ceiling_millicents)}; this suite's passes spent ${usdFromMillicents(res.suite_spent_last_day_millicents)} of ${usdFromMillicents(res.suite_daily_ceiling_millicents)} in the last 24h`
5040
+ );
5041
+ console.log(`Watch it with: erdo eval pass ${res.pass_id}`);
5031
5042
  console.log(
5032
5043
  `Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
5033
5044
  );
@@ -5035,6 +5046,19 @@ evalCmd.command("score <suite>").description("Score awaiting claims answers agai
5035
5046
  fail(e);
5036
5047
  }
5037
5048
  });
5049
+ evalCmd.command("pass <passId>").description("Show a claims pass's status and LLM spend so far (staff only)").action(async (passId) => {
5050
+ try {
5051
+ const p = await new ErdoClient().getEvalClaimsPass(passId);
5052
+ console.log(`pass: ${p.pass_id}`);
5053
+ console.log(`status: ${p.status}${p.resume_attempts > 0 ? ` (resumed ${p.resume_attempts}x)` : ""}`);
5054
+ console.log(`spend: ${usdFromMillicents(p.cost_millicents)} of ${usdFromMillicents(p.ceiling_millicents)} ceiling`);
5055
+ console.log(`scored: ${p.cases_scored} case(s), ${p.results_scored} answer(s)`);
5056
+ console.log(`started: ${p.started_at}${p.completed_at ? `, finished: ${p.completed_at}` : ""}`);
5057
+ if (p.error) console.log(`note: ${p.error}`);
5058
+ } catch (e) {
5059
+ fail(e);
5060
+ }
5061
+ });
5038
5062
  evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
5039
5063
  if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
5040
5064
  return parseInt(v, 10);
@@ -5255,7 +5279,7 @@ thread: ${res.thread_id}`);
5255
5279
  fail(e);
5256
5280
  }
5257
5281
  });
5258
- agentCmd.command("send <threadId> <message>").description("Send a message to a thread and print the agent's reply; message accepts @path to a UTF-8 file").option("-a, --agent <key>", "agent to invoke (e.g. erdo.artifact-builder)").option(
5282
+ agentCmd.command("send <threadId> <message>").description("Send a message to a thread and print the agent's reply; message accepts @path to a UTF-8 file").option("-a, --agent <key>", "agent to invoke (e.g. erdo.artifact-builder)").option("--model <id>", "model to use for this turn; empty uses the agent default").option(
5259
5283
  "--context <text>",
5260
5284
  "application context for this turn, or @path to a UTF-8 file; omitted from the visible user message"
5261
5285
  ).option(
@@ -5269,6 +5293,7 @@ agentCmd.command("send <threadId> <message>").description("Send a message to a t
5269
5293
  message: readMaybeFile(message) ?? "",
5270
5294
  context: readMaybeFile(opts.context),
5271
5295
  agent_key: opts.agent,
5296
+ model: opts.model,
5272
5297
  async: !opts.sync,
5273
5298
  read_only: opts.readOnly
5274
5299
  });
@@ -11020,9 +11045,15 @@ autoCmd.command("create <name>").description(
11020
11045
  }
11021
11046
  }
11022
11047
  );
11023
- autoCmd.command("list").description("List automations").option("--include-refresh", "Also list dataset refresh jobs with their state, including auto-paused ones and any that will pause soon").action(async (opts) => {
11024
- try {
11025
- const { heartbeats, refreshes } = await new ErdoClient().listHeartbeats({ includeRefresh: opts.includeRefresh });
11048
+ autoCmd.command("list").description("List automations").option("--include-refresh", "Also list dataset refresh jobs with their state, including auto-paused ones and any that will pause soon").option("--limit <n>", "Number of automations to return").option("--offset <n>", "Number of automations to skip", "0").action(async (opts) => {
11049
+ try {
11050
+ const rawLimit = opts.limit;
11051
+ const limit = rawLimit === void 0 ? void 0 : Number(rawLimit);
11052
+ const offset = Number(opts.offset);
11053
+ if (rawLimit !== void 0 && (!/^\d+$/.test(rawLimit) || limit === void 0 || limit < 1 || limit > 50)) throw new Error("limit must be an integer between 1 and 50");
11054
+ if (!/^\d+$/.test(opts.offset) || !Number.isInteger(offset) || offset < 0) throw new Error("offset must be a non-negative integer");
11055
+ const { heartbeats, refreshes, has_more } = await new ErdoClient().listHeartbeats({ includeRefresh: opts.includeRefresh, limit, offset: offset || void 0 });
11056
+ if (has_more) console.error(`more automations exist: rerun with --offset ${offset + heartbeats.length}`);
11026
11057
  for (const h of heartbeats) console.log(`${h.id} ${h.state} ${h.name} ${h.kind === "script" ? "script" : h.thread_id ? `thread:${h.thread_id}` : h.conversation ?? "new"}${h.persona_id ? ` persona:${h.persona_id}` : ""}${h.model ? ` model:${h.model}` : ""}${heartbeatBounds(h)}`);
11027
11058
  if (opts.includeRefresh) {
11028
11059
  for (const r of refreshes ?? []) console.log(formatRefreshJobRow(r));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.212.0",
3
+ "version": "0.214.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {