@erdoai/cli 0.201.0 → 0.203.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +115 -8
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -79,6 +79,50 @@ function formatRefreshJobRow(r) {
79
79
  if (r.keep_running) notes.push("keep-running");
80
80
  return `${r.id} ${r.state} ${r.name} refresh${notes.length ? ` ${notes.join("; ")}` : ""}`;
81
81
  }
82
+ var FRESHNESS_TARGET_MIN_SECONDS = 5 * 60;
83
+ var FRESHNESS_TARGET_MAX_SECONDS = 366 * 24 * 60 * 60;
84
+ var FRESHNESS_UNITS = { s: 1, m: 60, h: 60 * 60, d: 24 * 60 * 60, w: 7 * 24 * 60 * 60 };
85
+ function parseFreshnessTarget(value) {
86
+ const text = value.trim().toLowerCase();
87
+ if (text === "off" || text === "none" || text === "0") return 0;
88
+ let seconds;
89
+ if (/^\d+$/.test(text)) {
90
+ seconds = Number(text);
91
+ } else {
92
+ const parts = text.match(/^(?:\d+[smhdw])+$/) ? text.match(/\d+[smhdw]/g) : null;
93
+ if (!parts) {
94
+ throw new Error(`--freshness-target must be a duration like 30m, 2h, 1d or 1h30m, a number of seconds, or off (got ${JSON.stringify(value)})`);
95
+ }
96
+ seconds = parts.reduce((total, part) => total + Number(part.slice(0, -1)) * FRESHNESS_UNITS[part.slice(-1)], 0);
97
+ }
98
+ if (seconds === 0) return 0;
99
+ if (seconds < FRESHNESS_TARGET_MIN_SECONDS || seconds > FRESHNESS_TARGET_MAX_SECONDS) {
100
+ throw new Error("--freshness-target must be between 5m and 366d, or off to clear it");
101
+ }
102
+ return seconds;
103
+ }
104
+ function formatFreshnessTarget(seconds) {
105
+ const units = [[24 * 60 * 60, "day"], [60 * 60, "hour"], [60, "minute"]];
106
+ const plural = (n, unit) => `${n} ${unit}${n === 1 ? "" : "s"}`;
107
+ for (let i = 0; i < units.length; i++) {
108
+ const [size, name] = units[i];
109
+ if (seconds < size) continue;
110
+ let text = plural(Math.floor(seconds / size), name);
111
+ if (i + 1 < units.length) {
112
+ const minor = Math.floor(seconds % size / units[i + 1][0]);
113
+ if (minor > 0) text += ` ${plural(minor, units[i + 1][1])}`;
114
+ }
115
+ return text;
116
+ }
117
+ return plural(seconds, "second");
118
+ }
119
+ function describeFreshnessTarget(state) {
120
+ if (!state.freshness_target_seconds) return void 0;
121
+ const target = `data no older than ${formatFreshnessTarget(state.freshness_target_seconds)} \u2014 its owner is alerted when it falls behind, and it is never paused for being unread`;
122
+ if (!state.freshness_alert) return target;
123
+ const since = state.freshness_alert_raised_at ? ` since ${state.freshness_alert_raised_at}` : "";
124
+ return `${target}. ALERTING${since}: ${state.freshness_alert}`;
125
+ }
82
126
 
83
127
  // src/index.ts
84
128
  import { readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
@@ -150,7 +194,7 @@ function parseCSV(text) {
150
194
 
151
195
  // src/eval-compare.ts
152
196
  function evalResultScored(r) {
153
- return !r.agent_error && r.failure_classification !== "judge_error";
197
+ return !r.agent_error && r.failure_classification !== "judge_error" && r.scoring_status !== "awaiting_truth";
154
198
  }
155
199
  function compareEvalRuns(before, after, cases = []) {
156
200
  const nameByID = new Map(cases.map((c) => [c.id, c.name]));
@@ -1131,8 +1175,19 @@ var ErdoClient = class {
1131
1175
  }
1132
1176
  );
1133
1177
  }
1134
- getEvalLeaderboard(slug, days) {
1135
- const q = days && days > 0 ? `?days=${days}` : "";
1178
+ scoreEvalClaims(slug, input) {
1179
+ return this.request(
1180
+ "POST",
1181
+ `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
1182
+ input
1183
+ );
1184
+ }
1185
+ getEvalLeaderboard(slug, days, truthVersion) {
1186
+ const params = new URLSearchParams();
1187
+ if (days && days > 0) params.set("days", String(days));
1188
+ if (truthVersion && truthVersion > 0) params.set("truth_version", String(truthVersion));
1189
+ const qs = params.toString();
1190
+ const q = qs ? `?${qs}` : "";
1136
1191
  return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
1137
1192
  }
1138
1193
  getEvalRun(runID) {
@@ -3031,11 +3086,13 @@ function summariseResults(results, cases = []) {
3031
3086
  for (const r of results) {
3032
3087
  let status;
3033
3088
  if (r.agent_error) status = "ERROR";
3089
+ else if (r.scoring_status === "awaiting_truth") status = "AWAITING-TRUTH";
3034
3090
  else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
3035
3091
  else if (r.passed) status = "PASS";
3036
3092
  else status = "FAIL";
3037
3093
  const name = nameByID.get(r.case_id) ?? r.case_id;
3038
- console.log(` [${status}] ${r.score.toFixed(2)} ${name}`);
3094
+ const scoreText = r.scoring_status === "awaiting_truth" ? " -- " : r.score.toFixed(2);
3095
+ console.log(` [${status}] ${scoreText} ${name}`);
3039
3096
  if (r.candidate_cost_usd) console.log(` candidate cost: $${r.candidate_cost_usd.toFixed(4)} (reported by the candidate, not judge cost)`);
3040
3097
  if (r.agent_error) console.log(` agent error: ${r.agent_error}`);
3041
3098
  for (const c of r.judge_criteria_scores || []) {
@@ -3043,6 +3100,22 @@ function summariseResults(results, cases = []) {
3043
3100
  }
3044
3101
  }
3045
3102
  }
3103
+ function formatCriterionAverages(averages) {
3104
+ const withCriteria = (averages ?? []).filter((m) => m.criteria.length > 0);
3105
+ if (withCriteria.length === 0) return [];
3106
+ const lines = ["\nCriterion averages (avg score (n) per model)"];
3107
+ for (const m of withCriteria) {
3108
+ lines.push(` ${m.model}`);
3109
+ for (const c of m.criteria) lines.push(` ${c.avg_score.toFixed(2)} (${c.n}) ${c.criterion}`);
3110
+ }
3111
+ return lines;
3112
+ }
3113
+ function parseEvalTruthVersion(v) {
3114
+ const t = v.trim().toLowerCase();
3115
+ if (t === "new") return "new";
3116
+ if (/^[1-9][0-9]*$/.test(t)) return t;
3117
+ throw new Error(`--truth-version must be "new" or a positive integer, got "${v}"`);
3118
+ }
3046
3119
  function printEvalComparison(rows) {
3047
3120
  const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
3048
3121
  for (const r of rows) {
@@ -4779,7 +4852,7 @@ function parseStrictBoolean(value) {
4779
4852
  throw new Error("value must be true or false");
4780
4853
  }
4781
4854
  }
4782
- evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
4855
+ evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (empty string resets to default)").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
4783
4856
  async (slug, opts) => {
4784
4857
  try {
4785
4858
  const res = await new ErdoClient().updateEvalSuite(slug, {
@@ -4787,6 +4860,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
4787
4860
  description: opts.description,
4788
4861
  agent_key: opts.agent,
4789
4862
  judge_model: opts.judge,
4863
+ claims_verifier_model: opts.claimsVerifierModel,
4790
4864
  pass_threshold: opts.threshold,
4791
4865
  evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
4792
4866
  cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
@@ -4797,7 +4871,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
4797
4871
  }
4798
4872
  }
4799
4873
  );
4800
- evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
4874
+ evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (default if omitted)").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
4801
4875
  acc.push(v);
4802
4876
  return acc;
4803
4877
  }, []).action(
@@ -4809,6 +4883,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
4809
4883
  description: opts.description,
4810
4884
  agent_key: opts.agent,
4811
4885
  judge_model: opts.judge,
4886
+ claims_verifier_model: opts.claimsVerifierModel,
4812
4887
  pass_threshold: opts.threshold,
4813
4888
  evaluate_artifact: !!opts.evaluateArtifact,
4814
4889
  cron_enabled: opts.cron,
@@ -4899,9 +4974,27 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
4899
4974
  fail(e);
4900
4975
  }
4901
4976
  });
4902
- evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).action(async (slug, opts) => {
4977
+ evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
4903
4978
  try {
4904
- const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
4979
+ const truthVersion = parseEvalTruthVersion(opts.truthVersion);
4980
+ const res = await new ErdoClient().scoreEvalClaims(suite, {
4981
+ truth_version: truthVersion,
4982
+ regenerate_criteria: !!opts.regenerateCriteria
4983
+ });
4984
+ console.log(`pass: ${res.pass_id}`);
4985
+ console.log(
4986
+ `Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
4987
+ );
4988
+ } catch (e) {
4989
+ fail(e);
4990
+ }
4991
+ });
4992
+ evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
4993
+ if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
4994
+ return parseInt(v, 10);
4995
+ }).action(async (slug, opts) => {
4996
+ try {
4997
+ const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days, opts.truthVersion);
4905
4998
  if (res.models.length === 0) {
4906
4999
  console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
4907
5000
  return;
@@ -4933,6 +5026,7 @@ ${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`)
4933
5026
  }
4934
5027
  console.log(line);
4935
5028
  }
5029
+ for (const line of formatCriterionAverages(res.criterion_averages)) console.log(line);
4936
5030
  console.log(
4937
5031
  `
4938
5032
  Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
@@ -9078,6 +9172,8 @@ refreshCmd.command("show <slug>").description(
9078
9172
  const pendingPause = describePendingPause(res);
9079
9173
  if (pendingPause) rows.push(["pause warning", pendingPause]);
9080
9174
  if (res.keep_running) rows.push(["keep running", "yes \u2014 never paused for being unread"]);
9175
+ const freshness = describeFreshnessTarget(res);
9176
+ if (freshness) rows.push(["freshness target", freshness]);
9081
9177
  const eventTriggers = describeEventTriggers(res);
9082
9178
  if (eventTriggers) rows.push(["event triggers", eventTriggers]);
9083
9179
  if (res.schedule) {
@@ -9181,6 +9277,9 @@ refreshCmd.command("set <slug>").description(
9181
9277
  "--debounce <seconds>",
9182
9278
  "minimum gap between two view-triggered refreshes",
9183
9279
  (v) => parseInt(v, 10)
9280
+ ).option(
9281
+ "--freshness-target <duration>",
9282
+ "for a durable dataset people or systems rely on: the longest its data may go without a successful refresh or a write, e.g. 2h, 1d or 1h30m (5m to 366d). If it goes stale, or its refresh is paused or failing, the refresh's owner gets one attention item that resolves itself once the data is fresh again, and the refresh is never paused for being unread. Omit to keep the saved target; pass off to clear it"
9184
9283
  ).option(
9185
9284
  "--live-write-mode <append|replace>",
9186
9285
  "for the live strategies: append for a time-series that accumulates, replace for current state that overwrites on each event"
@@ -9290,6 +9389,9 @@ refreshCmd.command("set <slug>").description(
9290
9389
  body.refresh_on_view_stale_after_seconds = opts.staleAfter;
9291
9390
  }
9292
9391
  if (opts.debounce !== void 0) body.refresh_on_view_debounce_seconds = opts.debounce;
9392
+ if (opts.freshnessTarget !== void 0) {
9393
+ body.freshness_target_seconds = parseFreshnessTarget(opts.freshnessTarget);
9394
+ }
9293
9395
  if (opts.liveWriteMode) body.live_write_mode = opts.liveWriteMode;
9294
9396
  if (opts.payloadSchema) {
9295
9397
  body.payload_schema = readRefreshJSONFile(opts.payloadSchema, "--payload-schema");
@@ -9315,6 +9417,11 @@ refreshCmd.command("set <slug>").description(
9315
9417
  job_enabled: body.job_enabled,
9316
9418
  schedule_enabled: body.schedule_enabled
9317
9419
  }));
9420
+ if (res.freshness_target_seconds) {
9421
+ console.log(`freshness target: ${formatFreshnessTarget(res.freshness_target_seconds)} \u2014 its owner is alerted when the data falls behind`);
9422
+ } else if (body.freshness_target_seconds === 0) {
9423
+ console.log("freshness target: cleared");
9424
+ }
9318
9425
  if (res.job_id) console.log(`job: ${res.job_id}`);
9319
9426
  if (res.webhook_url) {
9320
9427
  console.log(`webhook url: ${res.webhook_url}`);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.201.0",
3
+ "version": "0.203.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {