@erdoai/cli 0.201.0 → 0.202.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +58 -8
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -150,7 +150,7 @@ function parseCSV(text) {
150
150
 
151
151
  // src/eval-compare.ts
152
152
  function evalResultScored(r) {
153
- return !r.agent_error && r.failure_classification !== "judge_error";
153
+ return !r.agent_error && r.failure_classification !== "judge_error" && r.scoring_status !== "awaiting_truth";
154
154
  }
155
155
  function compareEvalRuns(before, after, cases = []) {
156
156
  const nameByID = new Map(cases.map((c) => [c.id, c.name]));
@@ -1131,8 +1131,19 @@ var ErdoClient = class {
1131
1131
  }
1132
1132
  );
1133
1133
  }
1134
- getEvalLeaderboard(slug, days) {
1135
- const q = days && days > 0 ? `?days=${days}` : "";
1134
+ scoreEvalClaims(slug, input) {
1135
+ return this.request(
1136
+ "POST",
1137
+ `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
1138
+ input
1139
+ );
1140
+ }
1141
+ getEvalLeaderboard(slug, days, truthVersion) {
1142
+ const params = new URLSearchParams();
1143
+ if (days && days > 0) params.set("days", String(days));
1144
+ if (truthVersion && truthVersion > 0) params.set("truth_version", String(truthVersion));
1145
+ const qs = params.toString();
1146
+ const q = qs ? `?${qs}` : "";
1136
1147
  return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
1137
1148
  }
1138
1149
  getEvalRun(runID) {
@@ -3031,11 +3042,13 @@ function summariseResults(results, cases = []) {
3031
3042
  for (const r of results) {
3032
3043
  let status;
3033
3044
  if (r.agent_error) status = "ERROR";
3045
+ else if (r.scoring_status === "awaiting_truth") status = "AWAITING-TRUTH";
3034
3046
  else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
3035
3047
  else if (r.passed) status = "PASS";
3036
3048
  else status = "FAIL";
3037
3049
  const name = nameByID.get(r.case_id) ?? r.case_id;
3038
- console.log(` [${status}] ${r.score.toFixed(2)} ${name}`);
3050
+ const scoreText = r.scoring_status === "awaiting_truth" ? " -- " : r.score.toFixed(2);
3051
+ console.log(` [${status}] ${scoreText} ${name}`);
3039
3052
  if (r.candidate_cost_usd) console.log(` candidate cost: $${r.candidate_cost_usd.toFixed(4)} (reported by the candidate, not judge cost)`);
3040
3053
  if (r.agent_error) console.log(` agent error: ${r.agent_error}`);
3041
3054
  for (const c of r.judge_criteria_scores || []) {
@@ -3043,6 +3056,22 @@ function summariseResults(results, cases = []) {
3043
3056
  }
3044
3057
  }
3045
3058
  }
3059
+ function formatCriterionAverages(averages) {
3060
+ const withCriteria = (averages ?? []).filter((m) => m.criteria.length > 0);
3061
+ if (withCriteria.length === 0) return [];
3062
+ const lines = ["\nCriterion averages (avg score (n) per model)"];
3063
+ for (const m of withCriteria) {
3064
+ lines.push(` ${m.model}`);
3065
+ for (const c of m.criteria) lines.push(` ${c.avg_score.toFixed(2)} (${c.n}) ${c.criterion}`);
3066
+ }
3067
+ return lines;
3068
+ }
3069
+ function parseEvalTruthVersion(v) {
3070
+ const t = v.trim().toLowerCase();
3071
+ if (t === "new") return "new";
3072
+ if (/^[1-9][0-9]*$/.test(t)) return t;
3073
+ throw new Error(`--truth-version must be "new" or a positive integer, got "${v}"`);
3074
+ }
3046
3075
  function printEvalComparison(rows) {
3047
3076
  const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
3048
3077
  for (const r of rows) {
@@ -4779,7 +4808,7 @@ function parseStrictBoolean(value) {
4779
4808
  throw new Error("value must be true or false");
4780
4809
  }
4781
4810
  }
4782
- evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
4811
+ evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (empty string resets to default)").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
4783
4812
  async (slug, opts) => {
4784
4813
  try {
4785
4814
  const res = await new ErdoClient().updateEvalSuite(slug, {
@@ -4787,6 +4816,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
4787
4816
  description: opts.description,
4788
4817
  agent_key: opts.agent,
4789
4818
  judge_model: opts.judge,
4819
+ claims_verifier_model: opts.claimsVerifierModel,
4790
4820
  pass_threshold: opts.threshold,
4791
4821
  evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
4792
4822
  cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
@@ -4797,7 +4827,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
4797
4827
  }
4798
4828
  }
4799
4829
  );
4800
- evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
4830
+ evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (default if omitted)").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
4801
4831
  acc.push(v);
4802
4832
  return acc;
4803
4833
  }, []).action(
@@ -4809,6 +4839,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
4809
4839
  description: opts.description,
4810
4840
  agent_key: opts.agent,
4811
4841
  judge_model: opts.judge,
4842
+ claims_verifier_model: opts.claimsVerifierModel,
4812
4843
  pass_threshold: opts.threshold,
4813
4844
  evaluate_artifact: !!opts.evaluateArtifact,
4814
4845
  cron_enabled: opts.cron,
@@ -4899,9 +4930,27 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
4899
4930
  fail(e);
4900
4931
  }
4901
4932
  });
4902
- evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).action(async (slug, opts) => {
4933
+ evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
4934
+ try {
4935
+ const truthVersion = parseEvalTruthVersion(opts.truthVersion);
4936
+ const res = await new ErdoClient().scoreEvalClaims(suite, {
4937
+ truth_version: truthVersion,
4938
+ regenerate_criteria: !!opts.regenerateCriteria
4939
+ });
4940
+ console.log(`pass: ${res.pass_id}`);
4941
+ console.log(
4942
+ `Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
4943
+ );
4944
+ } catch (e) {
4945
+ fail(e);
4946
+ }
4947
+ });
4948
+ evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
4949
+ if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
4950
+ return parseInt(v, 10);
4951
+ }).action(async (slug, opts) => {
4903
4952
  try {
4904
- const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
4953
+ const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days, opts.truthVersion);
4905
4954
  if (res.models.length === 0) {
4906
4955
  console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
4907
4956
  return;
@@ -4933,6 +4982,7 @@ ${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`)
4933
4982
  }
4934
4983
  console.log(line);
4935
4984
  }
4985
+ for (const line of formatCriterionAverages(res.criterion_averages)) console.log(line);
4936
4986
  console.log(
4937
4987
  `
4938
4988
  Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.201.0",
3
+ "version": "0.202.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {