@erdoai/cli 0.201.0 → 0.203.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +115 -8
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -79,6 +79,50 @@ function formatRefreshJobRow(r) {
|
|
|
79
79
|
if (r.keep_running) notes.push("keep-running");
|
|
80
80
|
return `${r.id} ${r.state} ${r.name} refresh${notes.length ? ` ${notes.join("; ")}` : ""}`;
|
|
81
81
|
}
|
|
82
|
+
var FRESHNESS_TARGET_MIN_SECONDS = 5 * 60;
|
|
83
|
+
var FRESHNESS_TARGET_MAX_SECONDS = 366 * 24 * 60 * 60;
|
|
84
|
+
var FRESHNESS_UNITS = { s: 1, m: 60, h: 60 * 60, d: 24 * 60 * 60, w: 7 * 24 * 60 * 60 };
|
|
85
|
+
function parseFreshnessTarget(value) {
|
|
86
|
+
const text = value.trim().toLowerCase();
|
|
87
|
+
if (text === "off" || text === "none" || text === "0") return 0;
|
|
88
|
+
let seconds;
|
|
89
|
+
if (/^\d+$/.test(text)) {
|
|
90
|
+
seconds = Number(text);
|
|
91
|
+
} else {
|
|
92
|
+
const parts = text.match(/^(?:\d+[smhdw])+$/) ? text.match(/\d+[smhdw]/g) : null;
|
|
93
|
+
if (!parts) {
|
|
94
|
+
throw new Error(`--freshness-target must be a duration like 30m, 2h, 1d or 1h30m, a number of seconds, or off (got ${JSON.stringify(value)})`);
|
|
95
|
+
}
|
|
96
|
+
seconds = parts.reduce((total, part) => total + Number(part.slice(0, -1)) * FRESHNESS_UNITS[part.slice(-1)], 0);
|
|
97
|
+
}
|
|
98
|
+
if (seconds === 0) return 0;
|
|
99
|
+
if (seconds < FRESHNESS_TARGET_MIN_SECONDS || seconds > FRESHNESS_TARGET_MAX_SECONDS) {
|
|
100
|
+
throw new Error("--freshness-target must be between 5m and 366d, or off to clear it");
|
|
101
|
+
}
|
|
102
|
+
return seconds;
|
|
103
|
+
}
|
|
104
|
+
function formatFreshnessTarget(seconds) {
|
|
105
|
+
const units = [[24 * 60 * 60, "day"], [60 * 60, "hour"], [60, "minute"]];
|
|
106
|
+
const plural = (n, unit) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
|
107
|
+
for (let i = 0; i < units.length; i++) {
|
|
108
|
+
const [size, name] = units[i];
|
|
109
|
+
if (seconds < size) continue;
|
|
110
|
+
let text = plural(Math.floor(seconds / size), name);
|
|
111
|
+
if (i + 1 < units.length) {
|
|
112
|
+
const minor = Math.floor(seconds % size / units[i + 1][0]);
|
|
113
|
+
if (minor > 0) text += ` ${plural(minor, units[i + 1][1])}`;
|
|
114
|
+
}
|
|
115
|
+
return text;
|
|
116
|
+
}
|
|
117
|
+
return plural(seconds, "second");
|
|
118
|
+
}
|
|
119
|
+
function describeFreshnessTarget(state) {
|
|
120
|
+
if (!state.freshness_target_seconds) return void 0;
|
|
121
|
+
const target = `data no older than ${formatFreshnessTarget(state.freshness_target_seconds)} \u2014 its owner is alerted when it falls behind, and it is never paused for being unread`;
|
|
122
|
+
if (!state.freshness_alert) return target;
|
|
123
|
+
const since = state.freshness_alert_raised_at ? ` since ${state.freshness_alert_raised_at}` : "";
|
|
124
|
+
return `${target}. ALERTING${since}: ${state.freshness_alert}`;
|
|
125
|
+
}
|
|
82
126
|
|
|
83
127
|
// src/index.ts
|
|
84
128
|
import { readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
@@ -150,7 +194,7 @@ function parseCSV(text) {
|
|
|
150
194
|
|
|
151
195
|
// src/eval-compare.ts
|
|
152
196
|
function evalResultScored(r) {
|
|
153
|
-
return !r.agent_error && r.failure_classification !== "judge_error";
|
|
197
|
+
return !r.agent_error && r.failure_classification !== "judge_error" && r.scoring_status !== "awaiting_truth";
|
|
154
198
|
}
|
|
155
199
|
function compareEvalRuns(before, after, cases = []) {
|
|
156
200
|
const nameByID = new Map(cases.map((c) => [c.id, c.name]));
|
|
@@ -1131,8 +1175,19 @@ var ErdoClient = class {
|
|
|
1131
1175
|
}
|
|
1132
1176
|
);
|
|
1133
1177
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1178
|
+
scoreEvalClaims(slug, input) {
|
|
1179
|
+
return this.request(
|
|
1180
|
+
"POST",
|
|
1181
|
+
`/v1/evals/suites/${encodeURIComponent(slug)}/score-claims`,
|
|
1182
|
+
input
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
getEvalLeaderboard(slug, days, truthVersion) {
|
|
1186
|
+
const params = new URLSearchParams();
|
|
1187
|
+
if (days && days > 0) params.set("days", String(days));
|
|
1188
|
+
if (truthVersion && truthVersion > 0) params.set("truth_version", String(truthVersion));
|
|
1189
|
+
const qs = params.toString();
|
|
1190
|
+
const q = qs ? `?${qs}` : "";
|
|
1136
1191
|
return this.request("GET", `/v1/evals/leaderboard/${encodeURIComponent(slug)}${q}`);
|
|
1137
1192
|
}
|
|
1138
1193
|
getEvalRun(runID) {
|
|
@@ -3031,11 +3086,13 @@ function summariseResults(results, cases = []) {
|
|
|
3031
3086
|
for (const r of results) {
|
|
3032
3087
|
let status;
|
|
3033
3088
|
if (r.agent_error) status = "ERROR";
|
|
3089
|
+
else if (r.scoring_status === "awaiting_truth") status = "AWAITING-TRUTH";
|
|
3034
3090
|
else if (r.failure_classification === "judge_error") status = "JUDGE-ERROR";
|
|
3035
3091
|
else if (r.passed) status = "PASS";
|
|
3036
3092
|
else status = "FAIL";
|
|
3037
3093
|
const name = nameByID.get(r.case_id) ?? r.case_id;
|
|
3038
|
-
|
|
3094
|
+
const scoreText = r.scoring_status === "awaiting_truth" ? " -- " : r.score.toFixed(2);
|
|
3095
|
+
console.log(` [${status}] ${scoreText} ${name}`);
|
|
3039
3096
|
if (r.candidate_cost_usd) console.log(` candidate cost: $${r.candidate_cost_usd.toFixed(4)} (reported by the candidate, not judge cost)`);
|
|
3040
3097
|
if (r.agent_error) console.log(` agent error: ${r.agent_error}`);
|
|
3041
3098
|
for (const c of r.judge_criteria_scores || []) {
|
|
@@ -3043,6 +3100,22 @@ function summariseResults(results, cases = []) {
|
|
|
3043
3100
|
}
|
|
3044
3101
|
}
|
|
3045
3102
|
}
|
|
3103
|
+
function formatCriterionAverages(averages) {
|
|
3104
|
+
const withCriteria = (averages ?? []).filter((m) => m.criteria.length > 0);
|
|
3105
|
+
if (withCriteria.length === 0) return [];
|
|
3106
|
+
const lines = ["\nCriterion averages (avg score (n) per model)"];
|
|
3107
|
+
for (const m of withCriteria) {
|
|
3108
|
+
lines.push(` ${m.model}`);
|
|
3109
|
+
for (const c of m.criteria) lines.push(` ${c.avg_score.toFixed(2)} (${c.n}) ${c.criterion}`);
|
|
3110
|
+
}
|
|
3111
|
+
return lines;
|
|
3112
|
+
}
|
|
3113
|
+
function parseEvalTruthVersion(v) {
|
|
3114
|
+
const t = v.trim().toLowerCase();
|
|
3115
|
+
if (t === "new") return "new";
|
|
3116
|
+
if (/^[1-9][0-9]*$/.test(t)) return t;
|
|
3117
|
+
throw new Error(`--truth-version must be "new" or a positive integer, got "${v}"`);
|
|
3118
|
+
}
|
|
3046
3119
|
function printEvalComparison(rows) {
|
|
3047
3120
|
const fmt = (v) => v === null ? " \u2014 " : v.toFixed(2);
|
|
3048
3121
|
for (const r of rows) {
|
|
@@ -4779,7 +4852,7 @@ function parseStrictBoolean(value) {
|
|
|
4779
4852
|
throw new Error("value must be true or false");
|
|
4780
4853
|
}
|
|
4781
4854
|
}
|
|
4782
|
-
evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
|
|
4855
|
+
evalCmd.command("update <slug>").description("Update a suite's settings (only the flags you pass change)").option("--name <name>", "new name").option("--description <text>", "new description").option("--agent <key>", "repoint the agent under test, e.g. erdo.data-question-answerer").option("--judge <model>", "new judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (empty string resets to default)").option("--threshold <n>", "new pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact <bool>", "judge the rendered page (true/false)").option("--cron <bool>", "include in the daily cron (true/false)").action(
|
|
4783
4856
|
async (slug, opts) => {
|
|
4784
4857
|
try {
|
|
4785
4858
|
const res = await new ErdoClient().updateEvalSuite(slug, {
|
|
@@ -4787,6 +4860,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
|
|
|
4787
4860
|
description: opts.description,
|
|
4788
4861
|
agent_key: opts.agent,
|
|
4789
4862
|
judge_model: opts.judge,
|
|
4863
|
+
claims_verifier_model: opts.claimsVerifierModel,
|
|
4790
4864
|
pass_threshold: opts.threshold,
|
|
4791
4865
|
evaluate_artifact: opts.evaluateArtifact === void 0 ? void 0 : parseStrictBoolean(opts.evaluateArtifact),
|
|
4792
4866
|
cron_enabled: opts.cron === void 0 ? void 0 : parseStrictBoolean(opts.cron)
|
|
@@ -4797,7 +4871,7 @@ evalCmd.command("update <slug>").description("Update a suite's settings (only th
|
|
|
4797
4871
|
}
|
|
4798
4872
|
}
|
|
4799
4873
|
);
|
|
4800
|
-
evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
|
|
4874
|
+
evalCmd.command("create <name>").description("Create a suite (in the active org)").requiredOption("--agent <key>", "agent under test, e.g. erdo.artifact-builder").option("--description <text>", "description").option("--judge <model>", "judge model").option("--claims-verifier-model <model>", "model the claims verifier researches with; must differ from the judge (default if omitted)").option("--threshold <n>", "pass threshold >0 and <=5", parseEvalThreshold).option("--evaluate-artifact", "judge the rendered page (visual suite)").option("--no-cron", "exclude from the daily cron (recommended for artifact suites)").requiredOption("--case <json>", "first case: {name,input,rubric,tags?}", (v, acc) => {
|
|
4801
4875
|
acc.push(v);
|
|
4802
4876
|
return acc;
|
|
4803
4877
|
}, []).action(
|
|
@@ -4809,6 +4883,7 @@ evalCmd.command("create <name>").description("Create a suite (in the active org)
|
|
|
4809
4883
|
description: opts.description,
|
|
4810
4884
|
agent_key: opts.agent,
|
|
4811
4885
|
judge_model: opts.judge,
|
|
4886
|
+
claims_verifier_model: opts.claimsVerifierModel,
|
|
4812
4887
|
pass_threshold: opts.threshold,
|
|
4813
4888
|
evaluate_artifact: !!opts.evaluateArtifact,
|
|
4814
4889
|
cron_enabled: opts.cron,
|
|
@@ -4899,9 +4974,27 @@ ${run.status}: ${run.passed_cases}/${run.total_cases} passed, avg ${run.avg_scor
|
|
|
4899
4974
|
fail(e);
|
|
4900
4975
|
}
|
|
4901
4976
|
});
|
|
4902
|
-
evalCmd.command("
|
|
4977
|
+
evalCmd.command("score <suite>").description("Score awaiting claims answers against the truth (staff only); the pass runs in the background").option("--truth-version <n|new>", "truth version to score against (default: new)", "new").option("--regenerate-criteria", "write fresh criteria for a new truth version instead of carrying the previous version's forward").action(async (suite, opts) => {
|
|
4903
4978
|
try {
|
|
4904
|
-
const
|
|
4979
|
+
const truthVersion = parseEvalTruthVersion(opts.truthVersion);
|
|
4980
|
+
const res = await new ErdoClient().scoreEvalClaims(suite, {
|
|
4981
|
+
truth_version: truthVersion,
|
|
4982
|
+
regenerate_criteria: !!opts.regenerateCriteria
|
|
4983
|
+
});
|
|
4984
|
+
console.log(`pass: ${res.pass_id}`);
|
|
4985
|
+
console.log(
|
|
4986
|
+
`Read results with: erdo eval leaderboard ${suite} --truth-version <n>. A pass scores only answers from completed runs.`
|
|
4987
|
+
);
|
|
4988
|
+
} catch (e) {
|
|
4989
|
+
fail(e);
|
|
4990
|
+
}
|
|
4991
|
+
});
|
|
4992
|
+
evalCmd.command("leaderboard <slug>").description("Model-comparison table for a suite: cases as rows, models as columns").option("-d, --days <n>", "window in days (default: all time)", (v) => parseInt(v, 10)).option("--truth-version <n>", "only answers scored against this claims truth version", (v) => {
|
|
4993
|
+
if (!/^[1-9][0-9]*$/.test(v.trim())) throw new Error(`--truth-version must be a positive integer, got "${v}"`);
|
|
4994
|
+
return parseInt(v, 10);
|
|
4995
|
+
}).action(async (slug, opts) => {
|
|
4996
|
+
try {
|
|
4997
|
+
const res = await new ErdoClient().getEvalLeaderboard(slug, opts.days, opts.truthVersion);
|
|
4905
4998
|
if (res.models.length === 0) {
|
|
4906
4999
|
console.log(`No completed runs found for "${slug}"${opts.days ? ` in the last ${opts.days} days` : ""}.`);
|
|
4907
5000
|
return;
|
|
@@ -4933,6 +5026,7 @@ ${"Summary (avg of all cases)".padEnd(34)}${res.models.map(modelCol).join("")}`)
|
|
|
4933
5026
|
}
|
|
4934
5027
|
console.log(line);
|
|
4935
5028
|
}
|
|
5029
|
+
for (const line of formatCriterionAverages(res.criterion_averages)) console.log(line);
|
|
4936
5030
|
console.log(
|
|
4937
5031
|
`
|
|
4938
5032
|
Cell = avg score (pass rate) \xD7 runs. Cost is recorded judge cost per case; agent cost rides the run's tokens. "(unrec.)" = rows predating per-model capture.`
|
|
@@ -9078,6 +9172,8 @@ refreshCmd.command("show <slug>").description(
|
|
|
9078
9172
|
const pendingPause = describePendingPause(res);
|
|
9079
9173
|
if (pendingPause) rows.push(["pause warning", pendingPause]);
|
|
9080
9174
|
if (res.keep_running) rows.push(["keep running", "yes \u2014 never paused for being unread"]);
|
|
9175
|
+
const freshness = describeFreshnessTarget(res);
|
|
9176
|
+
if (freshness) rows.push(["freshness target", freshness]);
|
|
9081
9177
|
const eventTriggers = describeEventTriggers(res);
|
|
9082
9178
|
if (eventTriggers) rows.push(["event triggers", eventTriggers]);
|
|
9083
9179
|
if (res.schedule) {
|
|
@@ -9181,6 +9277,9 @@ refreshCmd.command("set <slug>").description(
|
|
|
9181
9277
|
"--debounce <seconds>",
|
|
9182
9278
|
"minimum gap between two view-triggered refreshes",
|
|
9183
9279
|
(v) => parseInt(v, 10)
|
|
9280
|
+
).option(
|
|
9281
|
+
"--freshness-target <duration>",
|
|
9282
|
+
"for a durable dataset people or systems rely on: the longest its data may go without a successful refresh or a write, e.g. 2h, 1d or 1h30m (5m to 366d). If it goes stale, or its refresh is paused or failing, the refresh's owner gets one attention item that resolves itself once the data is fresh again, and the refresh is never paused for being unread. Omit to keep the saved target; pass off to clear it"
|
|
9184
9283
|
).option(
|
|
9185
9284
|
"--live-write-mode <append|replace>",
|
|
9186
9285
|
"for the live strategies: append for a time-series that accumulates, replace for current state that overwrites on each event"
|
|
@@ -9290,6 +9389,9 @@ refreshCmd.command("set <slug>").description(
|
|
|
9290
9389
|
body.refresh_on_view_stale_after_seconds = opts.staleAfter;
|
|
9291
9390
|
}
|
|
9292
9391
|
if (opts.debounce !== void 0) body.refresh_on_view_debounce_seconds = opts.debounce;
|
|
9392
|
+
if (opts.freshnessTarget !== void 0) {
|
|
9393
|
+
body.freshness_target_seconds = parseFreshnessTarget(opts.freshnessTarget);
|
|
9394
|
+
}
|
|
9293
9395
|
if (opts.liveWriteMode) body.live_write_mode = opts.liveWriteMode;
|
|
9294
9396
|
if (opts.payloadSchema) {
|
|
9295
9397
|
body.payload_schema = readRefreshJSONFile(opts.payloadSchema, "--payload-schema");
|
|
@@ -9315,6 +9417,11 @@ refreshCmd.command("set <slug>").description(
|
|
|
9315
9417
|
job_enabled: body.job_enabled,
|
|
9316
9418
|
schedule_enabled: body.schedule_enabled
|
|
9317
9419
|
}));
|
|
9420
|
+
if (res.freshness_target_seconds) {
|
|
9421
|
+
console.log(`freshness target: ${formatFreshnessTarget(res.freshness_target_seconds)} \u2014 its owner is alerted when the data falls behind`);
|
|
9422
|
+
} else if (body.freshness_target_seconds === 0) {
|
|
9423
|
+
console.log("freshness target: cleared");
|
|
9424
|
+
}
|
|
9318
9425
|
if (res.job_id) console.log(`job: ${res.job_id}`);
|
|
9319
9426
|
if (res.webhook_url) {
|
|
9320
9427
|
console.log(`webhook url: ${res.webhook_url}`);
|