@erdoai/cli 0.200.0 → 0.202.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +183 -45
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -81,8 +81,8 @@ function formatRefreshJobRow(r) {
81
81
  }
82
82
 
83
83
  // src/index.ts
84
- import { readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "fs";
85
- import { basename } from "path";
84
+ import { readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
85
+ import { basename as basename2 } from "path";
86
86
 
87
87
  // src/csv.ts
88
88
  function parseCSV(text) {
@@ -150,7 +150,7 @@ function parseCSV(text) {
150
150
 
151
151
  // src/eval-compare.ts
152
152
  function evalResultScored(r) {
153
- return !r.agent_error && r.failure_classification !== "judge_error";
153
+ return !r.agent_error && r.failure_classification !== "judge_error" && r.scoring_status !== "awaiting_truth";
154
154
  }
155
155
  function compareEvalRuns(before, after, cases = []) {
156
156
  const nameByID = new Map(cases.map((c) => [c.id, c.name]));
@@ -177,25 +177,89 @@ function compareEvalRuns(before, after, cases = []) {
177
177
  return rows.sort((x, y) => (y.change ?? -Infinity) - (x.change ?? -Infinity));
178
178
  }
179
179
 
180
+ // src/eval-candidate.ts
181
+ import { readdirSync, readFileSync, statSync } from "fs";
182
+ import { extname, join, basename } from "path";
183
+ var ANSWER_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt"]);
184
+ var FILENAME_UNSAFE = /[\\/:*?"<>|]/g;
185
+ function caseKey(name) {
186
+ return name.trim().toLowerCase();
187
+ }
188
+ function fileSafeKey(name) {
189
+ return caseKey(name.replace(FILENAME_UNSAFE, "-"));
190
+ }
191
+ function loadCandidateOutputs(dir, caseNames, opts = {}) {
192
+ let entries;
193
+ try {
194
+ if (!statSync(dir).isDirectory()) throw new Error("not a directory");
195
+ entries = readdirSync(dir).sort();
196
+ } catch {
197
+ throw new Error(`--outputs must be a directory of answer files; cannot read "${dir}"`);
198
+ }
199
+ const byKey = /* @__PURE__ */ new Map();
200
+ const bySafeKey = /* @__PURE__ */ new Map();
201
+ for (const name of caseNames) {
202
+ byKey.set(caseKey(name), name);
203
+ bySafeKey.set(fileSafeKey(name), name);
204
+ }
205
+ const fileForCase = /* @__PURE__ */ new Map();
206
+ for (const entry of entries) {
207
+ const ext = extname(entry).toLowerCase();
208
+ if (entry.startsWith(".") || !ANSWER_EXTENSIONS.has(ext)) continue;
209
+ const stem = basename(entry, extname(entry));
210
+ const caseName = byKey.get(caseKey(stem)) ?? bySafeKey.get(fileSafeKey(stem));
211
+ if (caseName === void 0) {
212
+ throw new Error(
213
+ `"${entry}" matches no case in the suite. Name each file after its case (.md or .txt). Cases: ${caseNames.join(", ")}`
214
+ );
215
+ }
216
+ const existing = fileForCase.get(caseName);
217
+ if (existing !== void 0) {
218
+ throw new Error(`Case "${caseName}" has two answer files: "${existing}" and "${entry}". Keep one.`);
219
+ }
220
+ fileForCase.set(caseName, entry);
221
+ }
222
+ if (opts.requireAllCases) {
223
+ const missing = caseNames.filter((name) => !fileForCase.has(name));
224
+ if (missing.length > 0) {
225
+ throw new Error(
226
+ `No answer file for ${missing.length === 1 ? "case" : "cases"}: ${missing.map((n) => `"${n}"`).join(", ")} (expected "<case name>.md" or ".txt" in ${dir})`
227
+ );
228
+ }
229
+ }
230
+ if (fileForCase.size === 0) {
231
+ throw new Error(`No answer files found in "${dir}". Add one "<case name>.md" or ".txt" per case. Cases: ${caseNames.join(", ")}`);
232
+ }
233
+ const outputs = [];
234
+ for (const caseName of caseNames) {
235
+ const file = fileForCase.get(caseName);
236
+ if (file === void 0) continue;
237
+ const output = readFileSync(join(dir, file), "utf8");
238
+ if (output.trim() === "") throw new Error(`Answer file "${file}" for case "${caseName}" is empty`);
239
+ outputs.push({ case_name: caseName, output });
240
+ }
241
+ return outputs;
242
+ }
243
+
180
244
  // src/index.ts
181
245
  import { password, select } from "@inquirer/prompts";
182
246
  import { Argument, Command as Command2, Option as Option2 } from "commander";
183
247
 
184
248
  // src/config.ts
185
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
249
+ import { existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
186
250
  import { homedir } from "os";
187
- import { dirname, join } from "path";
251
+ import { dirname, join as join2 } from "path";
188
252
  var DEFAULT_API_URL = "https://api.erdo.ai";
189
- var CONFIG_DIR = join(process.env.XDG_CONFIG_HOME || join(homedir(), ".config"), "erdo");
190
- var CONFIG_PATH = join(CONFIG_DIR, "config.json");
253
+ var CONFIG_DIR = join2(process.env.XDG_CONFIG_HOME || join2(homedir(), ".config"), "erdo");
254
+ var CONFIG_PATH = join2(CONFIG_DIR, "config.json");
191
255
  var LEGACY_KEY = "default";
192
256
  function configPath(name) {
193
- return join(CONFIG_DIR, name);
257
+ return join2(CONFIG_DIR, name);
194
258
  }
195
259
  function loadConfigFile() {
196
260
  if (!existsSync(CONFIG_PATH)) return null;
197
261
  try {
198
- const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf8"));
262
+ const raw = JSON.parse(readFileSync2(CONFIG_PATH, "utf8"));
199
263
  if (raw && raw.version === 2 && raw.accounts) return raw;
200
264
  if (raw && typeof raw.token === "string") {
201
265
  return { version: 2, activeAccount: LEGACY_KEY, accounts: { [LEGACY_KEY]: raw } };
@@ -1051,7 +1115,7 @@ var ErdoClient = class {
1051
1115
  `/v1/evals/suites/${encodeURIComponent(slug)}/cases/${encodeURIComponent(caseName)}`
1052
1116
  );
1053
1117
  }
1054
- runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride, rotateCases) {
1118
+ runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride, rotateCases, externalCandidate) {
1055
1119
  return this.request(
1056
1120
  "POST",
1057
1121
  `/v1/evals/suites/${encodeURIComponent(slug)}/run`,
@@ -1062,12 +1126,24 @@ var ErdoClient = class {
1062
1126
  backend_commit_sha: backendCommitSha,
1063
1127
  source,
1064
1128
  model_override: modelOverride,
1065
- rotate_cases: rotateCases
1129
+ rotate_cases: rotateCases,
1130
+ external_candidate: externalCandidate
1066
1131
  }
1067
1132
  );
1068
1133
  }
1069
- getEvalLeaderboard(slug, days) {
1070
- const q = days && days > 0 ? `?days=${days}` : "";
1134
+ scoreEvalClaims(slug, input) {
1135
+ return this.request(
1136
+ "POST",
1137
+ `/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
1138
+ input
1139
+ );
1140
+ }
1141
+ getEvalLeaderboard(slug, days, truthVersion) {
1142
+ const params = new URLSearchParams();
1143
+ if (days && days > 0) params.set("days", String(days));
1144
+ if (truthVersion && truthVersion > 0) params.set("truth_version", String(truthVersion));
1145
+ const qs = params.toString();
1146
+ const q = qs ? `?${qs}` : "";
1071
1147
  return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
1072
1148
  }
1073
1149
  getEvalRun(runID) {
@@ -2564,11 +2640,11 @@ async function browserLogin() {
2564
2640
  }
2565
2641
 
2566
2642
  // src/input.ts
2567
- import { readFileSync as readFileSync2 } from "fs";
2643
+ import { readFileSync as readFileSync3 } from "fs";
2568
2644
  function readMaybeFile(value) {
2569
2645
  if (value === void 0) return void 0;
2570
2646
  if (value.startsWith("@@")) return value.slice(1);
2571
- return value.startsWith("@") ? readFileSync2(value.slice(1), "utf8") : value;
2647
+ return value.startsWith("@") ? readFileSync3(value.slice(1), "utf8") : value;
2572
2648
  }
2573
2649
  async function resolveSecretInput(options) {
2574
2650
  if (options.fromStdin) {
@@ -2703,7 +2779,7 @@ function orgOption(description) {
2703
2779
 
2704
2780
  // src/update.ts
2705
2781
  import { spawn as spawn2, spawnSync } from "child_process";
2706
- import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
2782
+ import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
2707
2783
  import { dirname as dirname2 } from "path";
2708
2784
  var PKG = "@erdoai/cli";
2709
2785
  var MANUAL_HINT = `npm install -g ${PKG}@latest`;
@@ -2741,7 +2817,7 @@ async function fetchLatestVersion(timeoutMs) {
2741
2817
  function readVersionCache() {
2742
2818
  try {
2743
2819
  if (!existsSync2(CACHE_PATH)) return {};
2744
- const raw = JSON.parse(readFileSync3(CACHE_PATH, "utf8"));
2820
+ const raw = JSON.parse(readFileSync4(CACHE_PATH, "utf8"));
2745
2821
  return raw && typeof raw === "object" ? raw : {};
2746
2822
  } catch {
2747
2823
  return {};
@@ -2809,7 +2885,7 @@ async function update(current) {
2809
2885
  }
2810
2886
 
2811
2887
  // src/index.ts
2812
- var pkg = JSON.parse(readFileSync4(new URL("../package.json", import.meta.url), "utf8"));
2888
+ var pkg = JSON.parse(readFileSync5(new URL("../package.json", import.meta.url), "utf8"));
2813
2889
  function fail(err) {
2814
2890
  if (err instanceof ErdoApiError) {
2815
2891
  console.error(`Error ${err.status}: ${err.message}`);
@@ -2893,7 +2969,7 @@ async function readAllStdin() {
2893
2969
  function readJSONObject(file, label) {
2894
2970
  let parsed;
2895
2971
  try {
2896
- parsed = JSON.parse(readFileSync4(file, "utf8"));
2972
+ parsed = JSON.parse(readFileSync5(file, "utf8"));
2897
2973
  } catch (error) {
2898
2974
  throw new Error(`${label} must be a readable JSON file: ${error instanceof Error ? error.message : String(error)}`);
2899
2975
  }
@@ -2966,17 +3042,36 @@ function summariseResults(results, cases = []) {
2966
3042
  for (const r of results) {
2967
3043
  let status;
2968
3044
  if (r.agent_error) status = "ERROR";
3045
+ else if (r.scoring_status === "awaiting_truth") status = "AWAITING-TRUTH";
2969
3046
  else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
2970
3047
  else if (r.passed) status = "PASS";
2971
3048
  else status = "FAIL";
2972
3049
  const name = nameByID.get(r.case_id) ?? r.case_id;
2973
- console.log(` [${status}] ${r.score.toFixed(2)} ${name}`);
3050
+ const scoreText = r.scoring_status === "awaiting_truth" ? " -- " : r.score.toFixed(2);
3051
+ console.log(` [${status}] ${scoreText} ${name}`);
3052
+ if (r.candidate_cost_usd) console.log(` candidate cost: $${r.candidate_cost_usd.toFixed(4)} (reported by the candidate, not judge cost)`);
2974
3053
  if (r.agent_error) console.log(` agent error: ${r.agent_error}`);
2975
3054
  for (const c of r.judge_criteria_scores || []) {
2976
3055
  console.log(` - ${c.score} ${c.criterion}`);
2977
3056
  }
2978
3057
  }
2979
3058
  }
3059
+ function formatCriterionAverages(averages) {
3060
+ const withCriteria = (averages ?? []).filter((m) => m.criteria.length > 0);
3061
+ if (withCriteria.length === 0) return [];
3062
+ const lines = ["\nCriterion averages (avg score (n) per model)"];
3063
+ for (const m of withCriteria) {
3064
+ lines.push(` ${m.model}`);
3065
+ for (const c of m.criteria) lines.push(` ${c.avg_score.toFixed(2)} (${c.n}) ${c.criterion}`);
3066
+ }
3067
+ return lines;
3068
+ }
3069
+ function parseEvalTruthVersion(v) {
3070
+ const t = v.trim().toLowerCase();
3071
+ if (t === "new") return "new";
3072
+ if (/^[1-9][0-9]*$/.test(t)) return t;
3073
+ throw new Error(`--truth-version must be "new" or a positive integer, got "${v}"`);
3074
+ }
2980
3075
  function printEvalComparison(rows) {
2981
3076
  const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
2982
3077
  for (const r of rows) {
@@ -4060,7 +4155,7 @@ evalCmd.command("suite <slug>").description("Show a suite and its cases").action
4060
4155
  var investigationCmd = program.command("investigation").description("Assess a data question from evidence already collected");
4061
4156
  investigationCmd.command("assess").description("Assess a question from a JSON file of definitions, observations, and claims").requiredOption("-f, --file <path>", "JSON assessment request").option("--json", "print the assessment JSON").action(async (opts) => {
4062
4157
  try {
4063
- const input = JSON.parse(readFileSync4(opts.file, "utf8"));
4158
+ const input = JSON.parse(readFileSync5(opts.file, "utf8"));
4064
4159
  const result = await new ErdoClient().assessInvestigation(input);
4065
4160
  if (opts.json) {
4066
4161
  print(result);
@@ -4247,7 +4342,7 @@ wsCmd.command("recommendations <slug>").description("Read resolved recommendatio
4247
4342
  try {
4248
4343
  const client = new ErdoClient();
4249
4344
  if (opts.inputFile) {
4250
- const body = JSON.parse(readFileSync4(opts.inputFile, "utf8"));
4345
+ const body = JSON.parse(readFileSync5(opts.inputFile, "utf8"));
4251
4346
  if (body === null || typeof body !== "object" || Array.isArray(body)) {
4252
4347
  throw new Error("--input-file must contain a JSON object");
4253
4348
  }
@@ -4713,7 +4808,7 @@ function parseStrictBoolean(value) {
4713
4808
  throw new Error("value must be true or false");
4714
4809
  }
4715
4810
  }
4716
- evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
4811
+ evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (empty string resets to default)").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
4717
4812
  async (slug, opts) => {
4718
4813
  try {
4719
4814
  const res = await new ErdoClient().updateEvalSuite(slug, {
@@ -4721,6 +4816,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
4721
4816
  description: opts.description,
4722
4817
  agent_key: opts.agent,
4723
4818
  judge_model: opts.judge,
4819
+ claims_verifier_model: opts.claimsVerifierModel,
4724
4820
  pass_threshold: opts.threshold,
4725
4821
  evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
4726
4822
  cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
@@ -4731,7 +4827,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
4731
4827
  }
4732
4828
  }
4733
4829
  );
4734
- evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
4830
+ evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (default if omitted)").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
4735
4831
  acc.push(v);
4736
4832
  return acc;
4737
4833
  }, []).action(
@@ -4743,6 +4839,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
4743
4839
  description: opts.description,
4744
4840
  agent_key: opts.agent,
4745
4841
  judge_model: opts.judge,
4842
+ claims_verifier_model: opts.claimsVerifierModel,
4746
4843
  pass_threshold: opts.threshold,
4747
4844
  evaluate_artifact: !!opts.evaluateArtifact,
4748
4845
  cron_enabled: opts.cron,
@@ -4761,10 +4858,28 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
4761
4858
  throw new Error(`--rotate-cases must be a non-negative integer, got "${v}"`);
4762
4859
  }
4763
4860
  return parsed;
4764
- }).action(async (slug, opts) => {
4861
+ }).option("--candidate <label>", "score another system's answers instead of invoking the agent; the label becomes the leaderboard column external:<label> (staff only; needs --outputs)").option("--outputs <dir>", `directory of the candidate's answers, one "<case name>.md" or ".txt" file per case (with --candidate)`).option("--candidate-notes <text>", "how the candidate's answers were captured, stored with the run (with --candidate)").option("--all-cases", "with --candidate, fail unless every case in the suite has an answer file").action(async (slug, opts) => {
4765
4862
  try {
4863
+ if (!opts.candidate && (opts.outputs || opts.candidateNotes || opts.allCases)) {
4864
+ throw new Error("--outputs, --candidate-notes and --all-cases need --candidate <label>");
4865
+ }
4866
+ if (opts.candidate && !opts.outputs) {
4867
+ throw new Error("--candidate needs --outputs <dir> holding one answer file per case");
4868
+ }
4869
+ if (opts.candidate && (opts.model || opts.rotateCases)) {
4870
+ throw new Error("--candidate cannot be combined with --model or --rotate-cases: the answers were not produced by an Erdo agent");
4871
+ }
4766
4872
  const api = new ErdoClient();
4767
- const { suite } = await api.getEvalSuite(slug);
4873
+ const { suite, cases: suiteCases } = await api.getEvalSuite(slug);
4874
+ const externalCandidate = opts.candidate ? {
4875
+ label: opts.candidate,
4876
+ outputs: loadCandidateOutputs(
4877
+ opts.outputs,
4878
+ suiteCases.map((c) => c.name),
4879
+ { requireAllCases: opts.allCases }
4880
+ ),
4881
+ notes: opts.candidateNotes
4882
+ } : void 0;
4768
4883
  if (suite.evaluate_artifact && !process.env.ERDO_ORG) {
4769
4884
  const resolved = resolveOrgId(activeAccount()) ?? "<org>";
4770
4885
  console.error(
@@ -4785,9 +4900,13 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
4785
4900
  opts.backendCommitSha,
4786
4901
  opts.source,
4787
4902
  opts.model,
4788
- opts.rotateCases
4903
+ opts.rotateCases,
4904
+ externalCandidate
4789
4905
  );
4790
4906
  console.log(`run_id: ${run_id}`);
4907
+ if (externalCandidate) {
4908
+ console.log(`scoring ${externalCandidate.outputs.length} answer(s) from external:${externalCandidate.label}`);
4909
+ }
4791
4910
  if (!opts.watch) return;
4792
4911
  for (; ; ) {
4793
4912
  await sleep2(5e3);
@@ -4811,9 +4930,27 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
4811
4930
  fail(e);
4812
4931
  }
4813
4932
  });
4814
- evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).action(async (slug, opts) => {
4933
+ evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
4934
+ try {
4935
+ const truthVersion = parseEvalTruthVersion(opts.truthVersion);
4936
+ const res = await new ErdoClient().scoreEvalClaims(suite, {
4937
+ truth_version: truthVersion,
4938
+ regenerate_criteria: !!opts.regenerateCriteria
4939
+ });
4940
+ console.log(`pass: ${res.pass_id}`);
4941
+ console.log(
4942
+ `Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
4943
+ );
4944
+ } catch (e) {
4945
+ fail(e);
4946
+ }
4947
+ });
4948
+ evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
4949
+ if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
4950
+ return parseInt(v, 10);
4951
+ }).action(async (slug, opts) => {
4815
4952
  try {
4816
- const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
4953
+ const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days, opts.truthVersion);
4817
4954
  if (res.models.length === 0) {
4818
4955
  console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
4819
4956
  return;
@@ -4845,6 +4982,7 @@ ${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`)
4845
4982
  }
4846
4983
  console.log(line);
4847
4984
  }
4985
+ for (const line of formatCriterionAverages(res.criterion_averages)) console.log(line);
4848
4986
  console.log(
4849
4987
  `
4850
4988
  Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
@@ -6564,7 +6702,7 @@ var pagesCmd = program.command("pages").description("Create and manage pages/art
6564
6702
  var criticCmd = program.command("critic").description("Review campaign creative");
6565
6703
  criticCmd.command("review").description("Review an ad creative request from a JSON file and print per-lens findings").requiredOption("-f, --file <path>", "JSON file containing the full critic review request envelope").option("--json", "print the complete raw JSON response").action(async (opts) => {
6566
6704
  try {
6567
- const input = JSON.parse(readFileSync4(opts.file, "utf8"));
6705
+ const input = JSON.parse(readFileSync5(opts.file, "utf8"));
6568
6706
  const result = await new ErdoClient().reviewAd(input);
6569
6707
  if (opts.json) {
6570
6708
  print(result);
@@ -8053,7 +8191,7 @@ function describeLOASignature(sig) {
8053
8191
  }
8054
8192
  }
8055
8193
  function fileAsBase64(path) {
8056
- return path ? readFileSync4(path).toString("base64") : void 0;
8194
+ return path ? readFileSync5(path).toString("base64") : void 0;
8057
8195
  }
8058
8196
  voiceBrandedCallingCmd.command("get").description("Show the brand, what it still needs, and each number's part in it").option("--json", "print the raw JSON result instead of a table").action(async (opts) => {
8059
8197
  try {
@@ -8328,7 +8466,7 @@ providerLeadSourcesCmd.command("import <name> <file>").description(
8328
8466
  try {
8329
8467
  if (opts.startFollowUp && !opts.commit) throw new Error("--start-follow-up needs --commit");
8330
8468
  if (!file.toLowerCase().endsWith(".csv")) throw new Error("import reads CSV; export the sheet or workbook as CSV first");
8331
- const records = parseCSV(readFileSync4(file, "utf8"));
8469
+ const records = parseCSV(readFileSync5(file, "utf8"));
8332
8470
  if (records.length === 0) throw new Error("the file is empty");
8333
8471
  const [header, ...rows] = records;
8334
8472
  const client = new ErdoClient();
@@ -8566,7 +8704,7 @@ datasetsCmd.command("write <slug>").description(
8566
8704
  }
8567
8705
  );
8568
8706
  function readSchemaFile(path) {
8569
- const parsed = JSON.parse(readFileSync4(path, "utf8"));
8707
+ const parsed = JSON.parse(readFileSync5(path, "utf8"));
8570
8708
  const schema = Array.isArray(parsed) ? { columns: parsed } : parsed;
8571
8709
  if (!schema || !Array.isArray(schema.columns)) {
8572
8710
  throw new Error('--schema must contain a JSON columns array or {"columns": [...]} object');
@@ -8630,7 +8768,7 @@ datasetsCmd.command("ensure <purpose>").description(
8630
8768
  });
8631
8769
  datasetsCmd.command("update-schema <slug>").description("Evolve an existing dataset's column contract. For populated lead-identity datasets, use add_column operations only; existing rows and lead ids are preserved.").requiredOption("--input <file>", "JSON request containing operations and optional class/purpose changes").action(async (slug, opts) => {
8632
8770
  try {
8633
- const input = JSON.parse(readFileSync4(opts.input, "utf8"));
8771
+ const input = JSON.parse(readFileSync5(opts.input, "utf8"));
8634
8772
  if (!input || typeof input !== "object" || Array.isArray(input)) {
8635
8773
  throw new Error("--input must contain a JSON object");
8636
8774
  }
@@ -8880,7 +9018,7 @@ Fetch a selected table with: erdo datasets fetch ${slug} --sql "SELECT * FROM <t
8880
9018
  var MAX_UPLOAD_BYTES = 20 * 1024 * 1024;
8881
9019
  datasetsCmd.command("upload <file>").description("Upload a file (CSV, Excel, JSON, ...) and create a dataset from it").option("-n, --name <name>", "display name for the dataset (defaults to the filename)").option("-d, --description <text>", "description shown to agents analyzing the dataset").action(async (file, opts) => {
8882
9020
  try {
8883
- const content = readFileSync4(file);
9021
+ const content = readFileSync5(file);
8884
9022
  if (content.byteLength > MAX_UPLOAD_BYTES) {
8885
9023
  fail(
8886
9024
  new Error(
@@ -8888,7 +9026,7 @@ datasetsCmd.command("upload <file>").description("Upload a file (CSV, Excel, JSO
8888
9026
  )
8889
9027
  );
8890
9028
  }
8891
- const filename = basename(file);
9029
+ const filename = basename2(file);
8892
9030
  const res = await new ErdoClient().uploadDatasetFile({
8893
9031
  filename,
8894
9032
  content_base64: content.toString("base64"),
@@ -9439,7 +9577,7 @@ Review saved submission evidence with: erdo leads resolve-capture ${res.dataset_
9439
9577
  });
9440
9578
  leadsCmd.command("resolve-capture <dataset> <receipt>").description("Preview a refused submission against a candidate lead, or record an audited dismissal. Applying preserves the original receipt and does not merge identities, report a conversion, or replay arrival automations.").requiredOption("--input <file>", "JSON decision: action, target_lead_id for apply_to_lead, reason; applying also requires expected_snapshot and operation_key").option("--apply", "confirm the exact preview; without this flag the decision only previews").action(async (dataset, receipt, opts) => {
9441
9579
  try {
9442
- const input = JSON.parse(readFileSync4(opts.input, "utf8"));
9580
+ const input = JSON.parse(readFileSync5(opts.input, "utf8"));
9443
9581
  if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("--input must contain a JSON object");
9444
9582
  if (Object.hasOwn(input, "apply")) throw new Error("Use the explicit --apply flag; do not put apply in the decision file");
9445
9583
  for (const key of Object.keys(input)) {
@@ -9558,7 +9696,7 @@ leadPlaybookCmd.command("set").description(
9558
9696
  if (opts.dailyContactCap !== void 0 && !Number.isInteger(opts.dailyContactCap)) {
9559
9697
  throw new Error("--daily-contact-cap must be a whole number");
9560
9698
  }
9561
- const body = readFileSync4(opts.file, "utf8");
9699
+ const body = readFileSync5(opts.file, "utf8");
9562
9700
  const res = await new ErdoClient().putLeadPlaybook({
9563
9701
  body,
9564
9702
  enabled: opts.enable ? true : opts.disable ? false : void 0,
@@ -9583,7 +9721,7 @@ leadPlaybookCmd.command("read-back").description(
9583
9721
  "read the draft back as saving it with that concierge would; leave it off to read it back with none, which is what saving with none does"
9584
9722
  ).option("--json", "print the raw JSON result instead of a summary").action(async (opts) => {
9585
9723
  try {
9586
- const res = await new ErdoClient().readLeadPlaybook(readFileSync4(opts.file, "utf8"), opts.agentId);
9724
+ const res = await new ErdoClient().readLeadPlaybook(readFileSync5(opts.file, "utf8"), opts.agentId);
9587
9725
  if (opts.json) {
9588
9726
  print(res);
9589
9727
  return;
@@ -9898,7 +10036,7 @@ leadNextActionsCmd.command("evaluate <dataset> <lead>").description(
9898
10036
  "Dry-run the playbook on one lead: show the decision Erdo would make now, with the email or handoff it would write, the rule it cites and why. Nothing is stored and nothing is sent. --file tries a draft playbook instead of the saved one."
9899
10037
  ).option("-f, --file <path>", "a draft playbook text to evaluate with instead of the saved one").option("--agent-id <id>", "try the draft as that concierge, from `erdo voice agents`; only read alongside --file").option("--json", "print the raw JSON result instead of a summary").action(async (dataset, lead, opts) => {
9900
10038
  try {
9901
- const body = opts.file ? readFileSync4(opts.file, "utf8") : void 0;
10039
+ const body = opts.file ? readFileSync5(opts.file, "utf8") : void 0;
9902
10040
  const res = await new ErdoClient().evaluateLeadNextAction(dataset, lead, body, opts.agentId);
9903
10041
  if (opts.json) {
9904
10042
  print(res);
@@ -10593,9 +10731,9 @@ knowledgeCmd.command("attach <objectId> <file>").description(
10593
10731
  ).option("--media-type <mime>", "MIME type hint, e.g. image/jpeg; detected from the filename otherwise").action(
10594
10732
  async (objectId, file, opts) => {
10595
10733
  try {
10596
- const body = opts.url ? { filename: basename(file), source_url: opts.url } : {
10597
- filename: basename(file),
10598
- content_base64: readFileSync4(file).toString("base64")
10734
+ const body = opts.url ? { filename: basename2(file), source_url: opts.url } : {
10735
+ filename: basename2(file),
10736
+ content_base64: readFileSync5(file).toString("base64")
10599
10737
  };
10600
10738
  print(
10601
10739
  await new ErdoClient().attachKnowledgeAsset(objectId, {
@@ -10654,7 +10792,7 @@ autoCmd.command("create <name>").description(
10654
10792
  fail(new Error("pass either --script-js or --script-file, not both"));
10655
10793
  return;
10656
10794
  }
10657
- const scriptJs = opts.scriptFile ? readFileSync4(opts.scriptFile, "utf8") : opts.scriptJs;
10795
+ const scriptJs = opts.scriptFile ? readFileSync5(opts.scriptFile, "utf8") : opts.scriptJs;
10658
10796
  const h = await new ErdoClient().createHeartbeat({
10659
10797
  name,
10660
10798
  description: opts.description,
@@ -10724,7 +10862,7 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
10724
10862
  fail(new Error("pass either --model or --clear-model, not both"));
10725
10863
  return;
10726
10864
  }
10727
- const scriptJs = opts.scriptFile ? readFileSync4(opts.scriptFile, "utf8") : opts.scriptJs;
10865
+ const scriptJs = opts.scriptFile ? readFileSync5(opts.scriptFile, "utf8") : opts.scriptJs;
10728
10866
  if (opts.at !== void 0 && opts.clearAt) {
10729
10867
  fail(new Error("pass either --at or --clear-at, not both"));
10730
10868
  return;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@erdoai/cli",
3
- "version": "0.200.0",
3
+ "version": "0.202.0",
4
4
  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
5
5
  "type": "module",
6
6
  "bin": {