@erdoai/cli 0.200.0 → 0.202.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +183 -45
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -81,8 +81,8 @@ function formatRefreshJobRow(r) {
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
// src/index.ts
|
|
84
|
-
import { readFileSync as
|
|
85
|
-
import { basename } from "path";
|
|
84
|
+
import { readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
85
|
+
import { basename as basename2 } from "path";
|
|
86
86
|
|
|
87
87
|
// src/csv.ts
|
|
88
88
|
function parseCSV(text) {
|
|
@@ -150,7 +150,7 @@ function parseCSV(text) {
|
|
|
150
150
|
|
|
151
151
|
// src/eval-compare.ts
|
|
152
152
|
function evalResultScored(r) {
|
|
153
|
-
return !r.agent_error && r.failure_classification !== "judge_error";
|
|
153
|
+
return !r.agent_error && r.failure_classification !== "judge_error" && r.scoring_status !== "awaiting_truth";
|
|
154
154
|
}
|
|
155
155
|
function compareEvalRuns(before, after, cases = []) {
|
|
156
156
|
const nameByID = new Map(cases.map((c) => [c.id, c.name]));
|
|
@@ -177,25 +177,89 @@ function compareEvalRuns(before, after, cases = []) {
|
|
|
177
177
|
return rows.sort((x, y) => (y.change ?? -Infinity) - (x.change ?? -Infinity));
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
+
// src/eval-candidate.ts
|
|
181
|
+
import { readdirSync, readFileSync, statSync } from "fs";
|
|
182
|
+
import { extname, join, basename } from "path";
|
|
183
|
+
var ANSWER_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt"]);
|
|
184
|
+
var FILENAME_UNSAFE = /[\\/:*?"<>|]/g;
|
|
185
|
+
function caseKey(name) {
|
|
186
|
+
return name.trim().toLowerCase();
|
|
187
|
+
}
|
|
188
|
+
function fileSafeKey(name) {
|
|
189
|
+
return caseKey(name.replace(FILENAME_UNSAFE, "-"));
|
|
190
|
+
}
|
|
191
|
+
function loadCandidateOutputs(dir, caseNames, opts = {}) {
|
|
192
|
+
let entries;
|
|
193
|
+
try {
|
|
194
|
+
if (!statSync(dir).isDirectory()) throw new Error("not a directory");
|
|
195
|
+
entries = readdirSync(dir).sort();
|
|
196
|
+
} catch {
|
|
197
|
+
throw new Error(`--outputs must be a directory of answer files; cannot read "${dir}"`);
|
|
198
|
+
}
|
|
199
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
200
|
+
const bySafeKey = /* @__PURE__ */ new Map();
|
|
201
|
+
for (const name of caseNames) {
|
|
202
|
+
byKey.set(caseKey(name), name);
|
|
203
|
+
bySafeKey.set(fileSafeKey(name), name);
|
|
204
|
+
}
|
|
205
|
+
const fileForCase = /* @__PURE__ */ new Map();
|
|
206
|
+
for (const entry of entries) {
|
|
207
|
+
const ext = extname(entry).toLowerCase();
|
|
208
|
+
if (entry.startsWith(".") || !ANSWER_EXTENSIONS.has(ext)) continue;
|
|
209
|
+
const stem = basename(entry, extname(entry));
|
|
210
|
+
const caseName = byKey.get(caseKey(stem)) ?? bySafeKey.get(fileSafeKey(stem));
|
|
211
|
+
if (caseName === void 0) {
|
|
212
|
+
throw new Error(
|
|
213
|
+
`"${entry}" matches no case in the suite. Name each file after its case (.md or .txt). Cases: ${caseNames.join(", ")}`
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
const existing = fileForCase.get(caseName);
|
|
217
|
+
if (existing !== void 0) {
|
|
218
|
+
throw new Error(`Case "${caseName}" has two answer files: "${existing}" and "${entry}". Keep one.`);
|
|
219
|
+
}
|
|
220
|
+
fileForCase.set(caseName, entry);
|
|
221
|
+
}
|
|
222
|
+
if (opts.requireAllCases) {
|
|
223
|
+
const missing = caseNames.filter((name) => !fileForCase.has(name));
|
|
224
|
+
if (missing.length > 0) {
|
|
225
|
+
throw new Error(
|
|
226
|
+
`No answer file for ${missing.length === 1 ? "case" : "cases"}: ${missing.map((n) => `"${n}"`).join(", ")} (expected "<case name>.md" or ".txt" in ${dir})`
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (fileForCase.size === 0) {
|
|
231
|
+
throw new Error(`No answer files found in "${dir}". Add one "<case name>.md" or ".txt" per case. Cases: ${caseNames.join(", ")}`);
|
|
232
|
+
}
|
|
233
|
+
const outputs = [];
|
|
234
|
+
for (const caseName of caseNames) {
|
|
235
|
+
const file = fileForCase.get(caseName);
|
|
236
|
+
if (file === void 0) continue;
|
|
237
|
+
const output = readFileSync(join(dir, file), "utf8");
|
|
238
|
+
if (output.trim() === "") throw new Error(`Answer file "${file}" for case "${caseName}" is empty`);
|
|
239
|
+
outputs.push({ case_name: caseName, output });
|
|
240
|
+
}
|
|
241
|
+
return outputs;
|
|
242
|
+
}
|
|
243
|
+
|
|
180
244
|
// src/index.ts
|
|
181
245
|
import { password, select } from "@inquirer/prompts";
|
|
182
246
|
import { Argument, Command as Command2, Option as Option2 } from "commander";
|
|
183
247
|
|
|
184
248
|
// src/config.ts
|
|
185
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
249
|
+
import { existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
186
250
|
import { homedir } from "os";
|
|
187
|
-
import { dirname, join } from "path";
|
|
251
|
+
import { dirname, join as join2 } from "path";
|
|
188
252
|
var DEFAULT_API_URL = "https://api.erdo.ai";
|
|
189
|
-
var CONFIG_DIR =
|
|
190
|
-
var CONFIG_PATH =
|
|
253
|
+
var CONFIG_DIR = join2(process.env.XDG_CONFIG_HOME || join2(homedir(), ".config"), "erdo");
|
|
254
|
+
var CONFIG_PATH = join2(CONFIG_DIR, "config.json");
|
|
191
255
|
var LEGACY_KEY = "default";
|
|
192
256
|
function configPath(name) {
|
|
193
|
-
return
|
|
257
|
+
return join2(CONFIG_DIR, name);
|
|
194
258
|
}
|
|
195
259
|
function loadConfigFile() {
|
|
196
260
|
if (!existsSync(CONFIG_PATH)) return null;
|
|
197
261
|
try {
|
|
198
|
-
const raw = JSON.parse(
|
|
262
|
+
const raw = JSON.parse(readFileSync2(CONFIG_PATH, "utf8"));
|
|
199
263
|
if (raw && raw.version === 2 && raw.accounts) return raw;
|
|
200
264
|
if (raw && typeof raw.token === "string") {
|
|
201
265
|
return { version: 2, activeAccount: LEGACY_KEY, accounts: { [LEGACY_KEY]: raw } };
|
|
@@ -1051,7 +1115,7 @@ var ErdoClient = class {
|
|
|
1051
1115
|
`/v1/evals/suites/${encodeURIComponent(slug)}/cases/${encodeURIComponent(caseName)}`
|
|
1052
1116
|
);
|
|
1053
1117
|
}
|
|
1054
|
-
runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride, rotateCases) {
|
|
1118
|
+
runEvalSuite(slug, concurrency, commitSha, frontendCommitSha, backendCommitSha, source, modelOverride, rotateCases, externalCandidate) {
|
|
1055
1119
|
return this.request(
|
|
1056
1120
|
"POST",
|
|
1057
1121
|
`/v1/evals/suites/${encodeURIComponent(slug)}/run`,
|
|
@@ -1062,12 +1126,24 @@ var ErdoClient = class {
|
|
|
1062
1126
|
backend_commit_sha: backendCommitSha,
|
|
1063
1127
|
source,
|
|
1064
1128
|
model_override: modelOverride,
|
|
1065
|
-
rotate_cases: rotateCases
|
|
1129
|
+
rotate_cases: rotateCases,
|
|
1130
|
+
external_candidate: externalCandidate
|
|
1066
1131
|
}
|
|
1067
1132
|
);
|
|
1068
1133
|
}
|
|
1069
|
-
|
|
1070
|
-
|
|
1134
|
+
scoreEvalClaims(slug, input) {
|
|
1135
|
+
return this.request(
|
|
1136
|
+
"POST",
|
|
1137
|
+
`/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
|
|
1138
|
+
input
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
getEvalLeaderboard(slug, days, truthVersion) {
|
|
1142
|
+
const params = new URLSearchParams();
|
|
1143
|
+
if (days && days > 0) params.set("days", String(days));
|
|
1144
|
+
if (truthVersion && truthVersion > 0) params.set("truth_version", String(truthVersion));
|
|
1145
|
+
const qs = params.toString();
|
|
1146
|
+
const q = qs ? `?${qs}` : "";
|
|
1071
1147
|
return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
|
|
1072
1148
|
}
|
|
1073
1149
|
getEvalRun(runID) {
|
|
@@ -2564,11 +2640,11 @@ async function browserLogin() {
|
|
|
2564
2640
|
}
|
|
2565
2641
|
|
|
2566
2642
|
// src/input.ts
|
|
2567
|
-
import { readFileSync as
|
|
2643
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
2568
2644
|
function readMaybeFile(value) {
|
|
2569
2645
|
if (value === void 0) return void 0;
|
|
2570
2646
|
if (value.startsWith("@@")) return value.slice(1);
|
|
2571
|
-
return value.startsWith("@") ?
|
|
2647
|
+
return value.startsWith("@") ? readFileSync3(value.slice(1), "utf8") : value;
|
|
2572
2648
|
}
|
|
2573
2649
|
async function resolveSecretInput(options) {
|
|
2574
2650
|
if (options.fromStdin) {
|
|
@@ -2703,7 +2779,7 @@ function orgOption(description) {
|
|
|
2703
2779
|
|
|
2704
2780
|
// src/update.ts
|
|
2705
2781
|
import { spawn as spawn2, spawnSync } from "child_process";
|
|
2706
|
-
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as
|
|
2782
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
2707
2783
|
import { dirname as dirname2 } from "path";
|
|
2708
2784
|
var PKG = "@erdoai/cli";
|
|
2709
2785
|
var MANUAL_HINT = `npm install -g ${PKG}@latest`;
|
|
@@ -2741,7 +2817,7 @@ async function fetchLatestVersion(timeoutMs) {
|
|
|
2741
2817
|
function readVersionCache() {
|
|
2742
2818
|
try {
|
|
2743
2819
|
if (!existsSync2(CACHE_PATH)) return {};
|
|
2744
|
-
const raw = JSON.parse(
|
|
2820
|
+
const raw = JSON.parse(readFileSync4(CACHE_PATH, "utf8"));
|
|
2745
2821
|
return raw && typeof raw === "object" ? raw : {};
|
|
2746
2822
|
} catch {
|
|
2747
2823
|
return {};
|
|
@@ -2809,7 +2885,7 @@ async function update(current) {
|
|
|
2809
2885
|
}
|
|
2810
2886
|
|
|
2811
2887
|
// src/index.ts
|
|
2812
|
-
var pkg = JSON.parse(
|
|
2888
|
+
var pkg = JSON.parse(readFileSync5(new URL("../package.json", import.meta.url), "utf8"));
|
|
2813
2889
|
function fail(err) {
|
|
2814
2890
|
if (err instanceof ErdoApiError) {
|
|
2815
2891
|
console.error(`Error ${err.status}: ${err.message}`);
|
|
@@ -2893,7 +2969,7 @@ async function readAllStdin() {
|
|
|
2893
2969
|
function readJSONObject(file, label) {
|
|
2894
2970
|
let parsed;
|
|
2895
2971
|
try {
|
|
2896
|
-
parsed = JSON.parse(
|
|
2972
|
+
parsed = JSON.parse(readFileSync5(file, "utf8"));
|
|
2897
2973
|
} catch (error) {
|
|
2898
2974
|
throw new Error(`${label} must be a readable JSON file: ${error instanceof Error ? error.message : String(error)}`);
|
|
2899
2975
|
}
|
|
@@ -2966,17 +3042,36 @@ function summariseResults(results, cases = []) {
|
|
|
2966
3042
|
for (const r of results) {
|
|
2967
3043
|
let status;
|
|
2968
3044
|
if (r.agent_error) status = "ERROR";
|
|
3045
|
+
else if (r.scoring_status === "awaiting_truth") status = "AWAITING-TRUTH";
|
|
2969
3046
|
else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
|
|
2970
3047
|
else if (r.passed) status = "PASS";
|
|
2971
3048
|
else status = "FAIL";
|
|
2972
3049
|
const name = nameByID.get(r.case_id) ?? r.case_id;
|
|
2973
|
-
|
|
3050
|
+
const scoreText = r.scoring_status === "awaiting_truth" ? " -- " : r.score.toFixed(2);
|
|
3051
|
+
console.log(` [${status}] ${scoreText} ${name}`);
|
|
3052
|
+
if (r.candidate_cost_usd) console.log(` candidate cost: $${r.candidate_cost_usd.toFixed(4)} (reported by the candidate, not judge cost)`);
|
|
2974
3053
|
if (r.agent_error) console.log(` agent error: ${r.agent_error}`);
|
|
2975
3054
|
for (const c of r.judge_criteria_scores || []) {
|
|
2976
3055
|
console.log(` - ${c.score} ${c.criterion}`);
|
|
2977
3056
|
}
|
|
2978
3057
|
}
|
|
2979
3058
|
}
|
|
3059
|
+
function formatCriterionAverages(averages) {
|
|
3060
|
+
const withCriteria = (averages ?? []).filter((m) => m.criteria.length > 0);
|
|
3061
|
+
if (withCriteria.length === 0) return [];
|
|
3062
|
+
const lines = ["\nCriterion averages (avg score (n) per model)"];
|
|
3063
|
+
for (const m of withCriteria) {
|
|
3064
|
+
lines.push(` ${m.model}`);
|
|
3065
|
+
for (const c of m.criteria) lines.push(` ${c.avg_score.toFixed(2)} (${c.n}) ${c.criterion}`);
|
|
3066
|
+
}
|
|
3067
|
+
return lines;
|
|
3068
|
+
}
|
|
3069
|
+
function parseEvalTruthVersion(v) {
|
|
3070
|
+
const t = v.trim().toLowerCase();
|
|
3071
|
+
if (t === "new") return "new";
|
|
3072
|
+
if (/^[1-9][0-9]*$/.test(t)) return t;
|
|
3073
|
+
throw new Error(`--truth-version must be "new" or a positive integer, got "${v}"`);
|
|
3074
|
+
}
|
|
2980
3075
|
function printEvalComparison(rows) {
|
|
2981
3076
|
const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
|
|
2982
3077
|
for (const r of rows) {
|
|
@@ -4060,7 +4155,7 @@ evalCmd.command("suite <slug>").description("Show a suite and its cases").action
|
|
|
4060
4155
|
var investigationCmd = program.command("investigation").description("Assess a data question from evidence already collected");
|
|
4061
4156
|
investigationCmd.command("assess").description("Assess a question from a JSON file of definitions, observations, and claims").requiredOption("-f, --file <path>", "JSON assessment request").option("--json", "print the assessment JSON").action(async (opts) => {
|
|
4062
4157
|
try {
|
|
4063
|
-
const input = JSON.parse(
|
|
4158
|
+
const input = JSON.parse(readFileSync5(opts.file, "utf8"));
|
|
4064
4159
|
const result = await new ErdoClient().assessInvestigation(input);
|
|
4065
4160
|
if (opts.json) {
|
|
4066
4161
|
print(result);
|
|
@@ -4247,7 +4342,7 @@ wsCmd.command("recommendations <slug>").description("Read resolved recommendatio
|
|
|
4247
4342
|
try {
|
|
4248
4343
|
const client = new ErdoClient();
|
|
4249
4344
|
if (opts.inputFile) {
|
|
4250
|
-
const body = JSON.parse(
|
|
4345
|
+
const body = JSON.parse(readFileSync5(opts.inputFile, "utf8"));
|
|
4251
4346
|
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
4252
4347
|
throw new Error("--input-file must contain a JSON object");
|
|
4253
4348
|
}
|
|
@@ -4713,7 +4808,7 @@ function parseStrictBoolean(value) {
|
|
|
4713
4808
|
throw new Error("value must be true or false");
|
|
4714
4809
|
}
|
|
4715
4810
|
}
|
|
4716
|
-
evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
|
|
4811
|
+
evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (empty string resets to default)").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
|
|
4717
4812
|
async (slug, opts) => {
|
|
4718
4813
|
try {
|
|
4719
4814
|
const res = await new ErdoClient().updateEvalSuite(slug, {
|
|
@@ -4721,6 +4816,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
|
|
|
4721
4816
|
description: opts.description,
|
|
4722
4817
|
agent_key: opts.agent,
|
|
4723
4818
|
judge_model: opts.judge,
|
|
4819
|
+
claims_verifier_model: opts.claimsVerifierModel,
|
|
4724
4820
|
pass_threshold: opts.threshold,
|
|
4725
4821
|
evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
|
|
4726
4822
|
cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
|
|
@@ -4731,7 +4827,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
|
|
|
4731
4827
|
}
|
|
4732
4828
|
}
|
|
4733
4829
|
);
|
|
4734
|
-
evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
|
|
4830
|
+
evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (default if omitted)").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
|
|
4735
4831
|
acc.push(v);
|
|
4736
4832
|
return acc;
|
|
4737
4833
|
}, []).action(
|
|
@@ -4743,6 +4839,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
|
|
|
4743
4839
|
description: opts.description,
|
|
4744
4840
|
agent_key: opts.agent,
|
|
4745
4841
|
judge_model: opts.judge,
|
|
4842
|
+
claims_verifier_model: opts.claimsVerifierModel,
|
|
4746
4843
|
pass_threshold: opts.threshold,
|
|
4747
4844
|
evaluate_artifact: !!opts.evaluateArtifact,
|
|
4748
4845
|
cron_enabled: opts.cron,
|
|
@@ -4761,10 +4858,28 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
|
|
|
4761
4858
|
throw new Error(`--rotate-cases must be a non-negative integer, got "${v}"`);
|
|
4762
4859
|
}
|
|
4763
4860
|
return parsed;
|
|
4764
|
-
}).action(async (slug, opts) => {
|
|
4861
|
+
}).option("--candidate <label>", "score another system's answers instead of invoking the agent; the label becomes the leaderboard column external:<label> (staff only; needs --outputs)").option("--outputs <dir>", `directory of the candidate's answers, one "<case name>.md" or ".txt" file per case (with --candidate)`).option("--candidate-notes <text>", "how the candidate's answers were captured, stored with the run (with --candidate)").option("--all-cases", "with --candidate, fail unless every case in the suite has an answer file").action(async (slug, opts) => {
|
|
4765
4862
|
try {
|
|
4863
|
+
if (!opts.candidate && (opts.outputs || opts.candidateNotes || opts.allCases)) {
|
|
4864
|
+
throw new Error("--outputs, --candidate-notes and --all-cases need --candidate <label>");
|
|
4865
|
+
}
|
|
4866
|
+
if (opts.candidate && !opts.outputs) {
|
|
4867
|
+
throw new Error("--candidate needs --outputs <dir> holding one answer file per case");
|
|
4868
|
+
}
|
|
4869
|
+
if (opts.candidate && (opts.model || opts.rotateCases)) {
|
|
4870
|
+
throw new Error("--candidate cannot be combined with --model or --rotate-cases: the answers were not produced by an Erdo agent");
|
|
4871
|
+
}
|
|
4766
4872
|
const api = new ErdoClient();
|
|
4767
|
-
const { suite } = await api.getEvalSuite(slug);
|
|
4873
|
+
const { suite, cases: suiteCases } = await api.getEvalSuite(slug);
|
|
4874
|
+
const externalCandidate = opts.candidate ? {
|
|
4875
|
+
label: opts.candidate,
|
|
4876
|
+
outputs: loadCandidateOutputs(
|
|
4877
|
+
opts.outputs,
|
|
4878
|
+
suiteCases.map((c) => c.name),
|
|
4879
|
+
{ requireAllCases: opts.allCases }
|
|
4880
|
+
),
|
|
4881
|
+
notes: opts.candidateNotes
|
|
4882
|
+
} : void 0;
|
|
4768
4883
|
if (suite.evaluate_artifact && !process.env.ERDO_ORG) {
|
|
4769
4884
|
const resolved = resolveOrgId(activeAccount()) ?? "<org>";
|
|
4770
4885
|
console.error(
|
|
@@ -4785,9 +4900,13 @@ evalCmd.command("run <slug>").description("Run a suite; --watch polls until it c
|
|
|
4785
4900
|
opts.backendCommitSha,
|
|
4786
4901
|
opts.source,
|
|
4787
4902
|
opts.model,
|
|
4788
|
-
opts.rotateCases
|
|
4903
|
+
opts.rotateCases,
|
|
4904
|
+
externalCandidate
|
|
4789
4905
|
);
|
|
4790
4906
|
console.log(`run_id: ${run_id}`);
|
|
4907
|
+
if (externalCandidate) {
|
|
4908
|
+
console.log(`scoring ${externalCandidate.outputs.length} answer(s) from external:${externalCandidate.label}`);
|
|
4909
|
+
}
|
|
4791
4910
|
if (!opts.watch) return;
|
|
4792
4911
|
for (; ; ) {
|
|
4793
4912
|
await sleep2(5e3);
|
|
@@ -4811,9 +4930,27 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
|
|
|
4811
4930
|
fail(e);
|
|
4812
4931
|
}
|
|
4813
4932
|
});
|
|
4814
|
-
evalCmd.command("
|
|
4933
|
+
evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
|
|
4934
|
+
try {
|
|
4935
|
+
const truthVersion = parseEvalTruthVersion(opts.truthVersion);
|
|
4936
|
+
const res = await new ErdoClient().scoreEvalClaims(suite, {
|
|
4937
|
+
truth_version: truthVersion,
|
|
4938
|
+
regenerate_criteria: !!opts.regenerateCriteria
|
|
4939
|
+
});
|
|
4940
|
+
console.log(`pass: ${res.pass_id}`);
|
|
4941
|
+
console.log(
|
|
4942
|
+
`Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
|
|
4943
|
+
);
|
|
4944
|
+
} catch (e) {
|
|
4945
|
+
fail(e);
|
|
4946
|
+
}
|
|
4947
|
+
});
|
|
4948
|
+
evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
|
|
4949
|
+
if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
|
|
4950
|
+
return parseInt(v, 10);
|
|
4951
|
+
}).action(async (slug, opts) => {
|
|
4815
4952
|
try {
|
|
4816
|
-
const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days);
|
|
4953
|
+
const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days, opts.truthVersion);
|
|
4817
4954
|
if (res.models.length === 0) {
|
|
4818
4955
|
console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
|
|
4819
4956
|
return;
|
|
@@ -4845,6 +4982,7 @@ ${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`)
|
|
|
4845
4982
|
}
|
|
4846
4983
|
console.log(line);
|
|
4847
4984
|
}
|
|
4985
|
+
for (const line of formatCriterionAverages(res.criterion_averages)) console.log(line);
|
|
4848
4986
|
console.log(
|
|
4849
4987
|
`
|
|
4850
4988
|
Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
|
|
@@ -6564,7 +6702,7 @@ var pagesCmd = program.command("pages").description("Create and manage pages/art
|
|
|
6564
6702
|
var criticCmd = program.command("critic").description("Review campaign creative");
|
|
6565
6703
|
criticCmd.command("review").description("Review an ad creative request from a JSON file and print per-lens findings").requiredOption("-f, --file <path>", "JSON file containing the full critic review request envelope").option("--json", "print the complete raw JSON response").action(async (opts) => {
|
|
6566
6704
|
try {
|
|
6567
|
-
const input = JSON.parse(
|
|
6705
|
+
const input = JSON.parse(readFileSync5(opts.file, "utf8"));
|
|
6568
6706
|
const result = await new ErdoClient().reviewAd(input);
|
|
6569
6707
|
if (opts.json) {
|
|
6570
6708
|
print(result);
|
|
@@ -8053,7 +8191,7 @@ function describeLOASignature(sig) {
|
|
|
8053
8191
|
}
|
|
8054
8192
|
}
|
|
8055
8193
|
function fileAsBase64(path) {
|
|
8056
|
-
return path ?
|
|
8194
|
+
return path ? readFileSync5(path).toString("base64") : void 0;
|
|
8057
8195
|
}
|
|
8058
8196
|
voiceBrandedCallingCmd.command("get").description("Show the brand, what it still needs, and each number's part in it").option("--json", "print the raw JSON result instead of a table").action(async (opts) => {
|
|
8059
8197
|
try {
|
|
@@ -8328,7 +8466,7 @@ providerLeadSourcesCmd.command("import <name> <file>").description(
|
|
|
8328
8466
|
try {
|
|
8329
8467
|
if (opts.startFollowUp && !opts.commit) throw new Error("--start-follow-up needs --commit");
|
|
8330
8468
|
if (!file.toLowerCase().endsWith(".csv")) throw new Error("import reads CSV; export the sheet or workbook as CSV first");
|
|
8331
|
-
const records = parseCSV(
|
|
8469
|
+
const records = parseCSV(readFileSync5(file, "utf8"));
|
|
8332
8470
|
if (records.length === 0) throw new Error("the file is empty");
|
|
8333
8471
|
const [header, ...rows] = records;
|
|
8334
8472
|
const client = new ErdoClient();
|
|
@@ -8566,7 +8704,7 @@ datasetsCmd.command("write <slug>").description(
|
|
|
8566
8704
|
}
|
|
8567
8705
|
);
|
|
8568
8706
|
function readSchemaFile(path) {
|
|
8569
|
-
const parsed = JSON.parse(
|
|
8707
|
+
const parsed = JSON.parse(readFileSync5(path, "utf8"));
|
|
8570
8708
|
const schema = Array.isArray(parsed) ? { columns: parsed } : parsed;
|
|
8571
8709
|
if (!schema || !Array.isArray(schema.columns)) {
|
|
8572
8710
|
throw new Error('--schema must contain a JSON columns array or {"columns": [...]} object');
|
|
@@ -8630,7 +8768,7 @@ datasetsCmd.command("ensure <purpose>").description(
|
|
|
8630
8768
|
});
|
|
8631
8769
|
datasetsCmd.command("update-schema <slug>").description("Evolve an existing dataset's column contract. For populated lead-identity datasets, use add_column operations only; existing rows and lead ids are preserved.").requiredOption("--input <file>", "JSON request containing operations and optional class/purpose changes").action(async (slug, opts) => {
|
|
8632
8770
|
try {
|
|
8633
|
-
const input = JSON.parse(
|
|
8771
|
+
const input = JSON.parse(readFileSync5(opts.input, "utf8"));
|
|
8634
8772
|
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
8635
8773
|
throw new Error("--input must contain a JSON object");
|
|
8636
8774
|
}
|
|
@@ -8880,7 +9018,7 @@ Fetch a selected table with: erdo datasets fetch ${slug} --sql "SELECT * FROM <t
|
|
|
8880
9018
|
var MAX_UPLOAD_BYTES = 20 * 1024 * 1024;
|
|
8881
9019
|
datasetsCmd.command("upload <file>").description("Upload a file (CSV, Excel, JSON, ...) and create a dataset from it").option("-n, --name <name>", "display name for the dataset (defaults to the filename)").option("-d, --description <text>", "description shown to agents analyzing the dataset").action(async (file, opts) => {
|
|
8882
9020
|
try {
|
|
8883
|
-
const content =
|
|
9021
|
+
const content = readFileSync5(file);
|
|
8884
9022
|
if (content.byteLength > MAX_UPLOAD_BYTES) {
|
|
8885
9023
|
fail(
|
|
8886
9024
|
new Error(
|
|
@@ -8888,7 +9026,7 @@ datasetsCmd.command("upload <file>").description("Upload a file (CSV, Excel, JSO
|
|
|
8888
9026
|
)
|
|
8889
9027
|
);
|
|
8890
9028
|
}
|
|
8891
|
-
const filename =
|
|
9029
|
+
const filename = basename2(file);
|
|
8892
9030
|
const res = await new ErdoClient().uploadDatasetFile({
|
|
8893
9031
|
filename,
|
|
8894
9032
|
content_base64: content.toString("base64"),
|
|
@@ -9439,7 +9577,7 @@ Review saved submission evidence with: erdo leads resolve-capture ${res.dataset_
|
|
|
9439
9577
|
});
|
|
9440
9578
|
leadsCmd.command("resolve-capture <dataset> <receipt>").description("Preview a refused submission against a candidate lead, or record an audited dismissal. Applying preserves the original receipt and does not merge identities, report a conversion, or replay arrival automations.").requiredOption("--input <file>", "JSON decision: action, target_lead_id for apply_to_lead, reason; applying also requires expected_snapshot and operation_key").option("--apply", "confirm the exact preview; without this flag the decision only previews").action(async (dataset, receipt, opts) => {
|
|
9441
9579
|
try {
|
|
9442
|
-
const input = JSON.parse(
|
|
9580
|
+
const input = JSON.parse(readFileSync5(opts.input, "utf8"));
|
|
9443
9581
|
if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("--input must contain a JSON object");
|
|
9444
9582
|
if (Object.hasOwn(input, "apply")) throw new Error("Use the explicit --apply flag; do not put apply in the decision file");
|
|
9445
9583
|
for (const key of Object.keys(input)) {
|
|
@@ -9558,7 +9696,7 @@ leadPlaybookCmd.command("set").description(
|
|
|
9558
9696
|
if (opts.dailyContactCap !== void 0 && !Number.isInteger(opts.dailyContactCap)) {
|
|
9559
9697
|
throw new Error("--daily-contact-cap must be a whole number");
|
|
9560
9698
|
}
|
|
9561
|
-
const body =
|
|
9699
|
+
const body = readFileSync5(opts.file, "utf8");
|
|
9562
9700
|
const res = await new ErdoClient().putLeadPlaybook({
|
|
9563
9701
|
body,
|
|
9564
9702
|
enabled: opts.enable ? true : opts.disable ? false : void 0,
|
|
@@ -9583,7 +9721,7 @@ leadPlaybookCmd.command("read-back").description(
|
|
|
9583
9721
|
"read the draft back as saving it with that concierge would; leave it off to read it back with none, which is what saving with none does"
|
|
9584
9722
|
).option("--json", "print the raw JSON result instead of a summary").action(async (opts) => {
|
|
9585
9723
|
try {
|
|
9586
|
-
const res = await new ErdoClient().readLeadPlaybook(
|
|
9724
|
+
const res = await new ErdoClient().readLeadPlaybook(readFileSync5(opts.file, "utf8"), opts.agentId);
|
|
9587
9725
|
if (opts.json) {
|
|
9588
9726
|
print(res);
|
|
9589
9727
|
return;
|
|
@@ -9898,7 +10036,7 @@ leadNextActionsCmd.command("evaluate <dataset> <lead>").description(
|
|
|
9898
10036
|
"Dry-run the playbook on one lead: show the decision Erdo would make now, with the email or handoff it would write, the rule it cites and why. Nothing is stored and nothing is sent. --file tries a draft playbook instead of the saved one."
|
|
9899
10037
|
).option("-f, --file <path>", "a draft playbook text to evaluate with instead of the saved one").option("--agent-id <id>", "try the draft as that concierge, from `erdo voice agents`; only read alongside --file").option("--json", "print the raw JSON result instead of a summary").action(async (dataset, lead, opts) => {
|
|
9900
10038
|
try {
|
|
9901
|
-
const body = opts.file ?
|
|
10039
|
+
const body = opts.file ? readFileSync5(opts.file, "utf8") : void 0;
|
|
9902
10040
|
const res = await new ErdoClient().evaluateLeadNextAction(dataset, lead, body, opts.agentId);
|
|
9903
10041
|
if (opts.json) {
|
|
9904
10042
|
print(res);
|
|
@@ -10593,9 +10731,9 @@ knowledgeCmd.command("attach <objectId> <file>").description(
|
|
|
10593
10731
|
).option("--media-type <mime>", "MIME type hint, e.g. image/jpeg; detected from the filename otherwise").action(
|
|
10594
10732
|
async (objectId, file, opts) => {
|
|
10595
10733
|
try {
|
|
10596
|
-
const body = opts.url ? { filename:
|
|
10597
|
-
filename:
|
|
10598
|
-
content_base64:
|
|
10734
|
+
const body = opts.url ? { filename: basename2(file), source_url: opts.url } : {
|
|
10735
|
+
filename: basename2(file),
|
|
10736
|
+
content_base64: readFileSync5(file).toString("base64")
|
|
10599
10737
|
};
|
|
10600
10738
|
print(
|
|
10601
10739
|
await new ErdoClient().attachKnowledgeAsset(objectId, {
|
|
@@ -10654,7 +10792,7 @@ autoCmd.command("create <name>").description(
|
|
|
10654
10792
|
fail(new Error("pass either --script-js or --script-file, not both"));
|
|
10655
10793
|
return;
|
|
10656
10794
|
}
|
|
10657
|
-
const scriptJs = opts.scriptFile ?
|
|
10795
|
+
const scriptJs = opts.scriptFile ? readFileSync5(opts.scriptFile, "utf8") : opts.scriptJs;
|
|
10658
10796
|
const h = await new ErdoClient().createHeartbeat({
|
|
10659
10797
|
name,
|
|
10660
10798
|
description: opts.description,
|
|
@@ -10724,7 +10862,7 @@ autoCmd.command("update <id>").description("Edit an automation \u2014 rename, re
|
|
|
10724
10862
|
fail(new Error("pass either --model or --clear-model, not both"));
|
|
10725
10863
|
return;
|
|
10726
10864
|
}
|
|
10727
|
-
const scriptJs = opts.scriptFile ?
|
|
10865
|
+
const scriptJs = opts.scriptFile ? readFileSync5(opts.scriptFile, "utf8") : opts.scriptJs;
|
|
10728
10866
|
if (opts.at !== void 0 && opts.clearAt) {
|
|
10729
10867
|
fail(new Error("pass either --at or --clear-at, not both"));
|
|
10730
10868
|
return;
|