@evolvingmachines/sdk 0.0.52-project-sable.20260729.cc76df1 → 0.0.52-project-sable.20260729.74a23a6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hosted/cli.cjs +2 -2
- package/dist/hosted/cli.d.cts +7 -2
- package/dist/hosted/cli.d.ts +7 -2
- package/dist/hosted/cli.js +4 -4
- package/dist/index.cjs +46 -46
- package/dist/index.d.cts +79 -5
- package/dist/index.d.ts +79 -5
- package/dist/index.js +46 -46
- package/dist/{types-Dsz--evy.d.cts → types-CMNbgLpL.d.cts} +10 -8
- package/dist/{types-Dsz--evy.d.ts → types-CMNbgLpL.d.ts} +10 -8
- package/harness-capabilities.json +345 -0
- package/package.json +12 -8
package/dist/hosted/cli.cjs
CHANGED
|
@@ -84,7 +84,7 @@ Other options:
|
|
|
84
84
|
--api-key <key> API key (default: $EVOLVE_API_KEY)
|
|
85
85
|
--base-url <url> API base URL (default: the Evolve dashboard API)`,m=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},gt={json:"boolean","api-key":"string","base-url":"string"},pt={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},Se={command:"help",positionals:[],flags:{}};function ft(e){if(e.length===0)throw new m("No command given");if(e[0]==="--help"||e[0]==="-h")return Se;let t=e[0],r=pt[t];if(!r)throw new m(`Unknown command "${t}"`);let n={},s=[];for(let i=1;i<e.length;i++){let o=e[i];if(o==="--help"||o==="-h")return Se;if(!o.startsWith("--")){s.push(o);continue}let u=o.slice(2),a,l=u.indexOf("=");l!==-1&&(a=u.slice(l+1),u=u.slice(0,l));let d=r.flags[u]??gt[u];if(!d)throw new m(`Unknown option --${u} for "${t}"`);if(d==="boolean"){if(a!==void 0)throw new m(`Option --${u} takes no value`);n[u]=true;continue}let c=a;if(c===void 0){let p=e[i+1];if(p===void 0||p.startsWith("--"))throw new m(`Option --${u} requires a value`);c=p,i++;}if(d==="number"){let p=Number(c);if(c.trim()===""||!Number.isFinite(p))throw new m(`Option --${u} expects a number, got "${c}"`);n[u]=p;}else if(d==="repeat"){let p=n[u]??[];p.push(c),n[u]=p;}else n[u]=c;}if(s.length<r.minPositionals)throw new m(`"${t}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(s.length>r.maxPositionals)throw new m(`"${t}" got unexpected argument "${s[r.maxPositionals]}"`);for(let i of r.required??[])if(!(i in n))throw new m(`"${t}" requires --${i}`);return {command:t,positionals:s,flags:n}}function bt(e){let t=e.indexOf(":");if(t<=0||t===e.length-1)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);let r=e.slice(0,t),n=e.slice(t+1),s=n.indexOf(":");if(s===-1)return {harness:r,model:n};let i=n.slice(0,s),o=n.slice(s+1);if(!i||!o)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);return {harness:r,model:i,harnessVersion:o}}function ht(e){let t=e.flags,r;if(t.tasks!==void 0&&(r=String(t.tasks).split(",").map(n=>n.trim()).filter(Boolean),r.length===0))throw new m("--tasks got an empty task list");return {benchmark:t.benchmark,...r!==void 0?{tasks:r}:{},agents:t.agent.map(n=>{let s=bt(n);return t.effort!==void 0?{...s,reasoningEffort:t.effort}:s}),...t.runs!==void 0?{runsPerTask:t.runs}:{},...t.concurrency!==void 0?{concurrency:t.concurrency}:{},...t["max-trial-spend"]!==void 0?{maxTrialSpendUsd:t["max-trial-spend"]}:{},...t.provider!==void 0?{sandboxProvider:t.provider}:{}}}function yt(e){let t=e.flags,r=typeof t.dir=="string",n=typeof t.git=="string"||typeof t.ref=="string";if(r&&n)throw new m('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let s of ["name","version"])if(typeof t[s]!="string")throw new m(`"import" requires --${s}`);return {source:{directory:t.dir},benchmarkName:t.name,version:t.version}}for(let s of ["git","ref","name","version"])if(typeof t[s]!="string"){let i=s==="git"||s==="ref"?" (or --dir for a local corpus directory)":"";throw new m(`"import" requires --${s}${i}`)}return {source:{gitUrl:t.git,ref:t.ref},benchmarkName:t.name,version:t.version}}function kt(e){let t={};for(let r of e){let n=r.indexOf("=");if(n<=0)throw new m(`Invalid --env "${r}": expected KEY=VALUE`);t[r.slice(0,n)]=r.slice(n+1);}return t}function Rt(e,t=r=>fs.readFileSync(r,"utf-8")){let r=e.flags,n=typeof r.dir=="string",s=typeof r["install-script"]=="string";if(n&&s)throw new m('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!s)throw new m('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let o of ["name","run"])if(typeof r[o]!="string")throw new m(`"custom-harnesses add" requires --${o}`);let i=kt(r.env??[]);return {name:r.name,...n?{directory:r.dir}:{installScript:t(r["install-script"])},runCommand:r.run,...Object.keys(i).length>0?{env:i}:{}}}var vt={out:e=>process.stdout.write(e+`
|
|
86
86
|
`),err:e=>process.stderr.write(e+`
|
|
87
|
-
`)};function h(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join(" ").trimEnd())}function T(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function se(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function D(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(se).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",T(e.maxTrialSpendUsd)]),t.push(["worst case",T(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",T(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function wt(e){return [e.id,e.status,e.benchmark,String(e.trials.total),$(e.meanReward),T(e.spentUsd),e.createdAt]}function St(e){return [e.taskKey,se(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",T(e.spentUsd),e.id]}function xt(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",se(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];
|
|
87
|
+
`)};function h(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join(" ").trimEnd())}function T(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function se(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function D(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(se).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",T(e.maxTrialSpendUsd)]),t.push(["worst case",T(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",T(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function wt(e){return [e.id,e.status,e.benchmark,String(e.trials.total),$(e.meanReward),T(e.spentUsd),e.createdAt]}function St(e){return [e.taskKey,se(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",T(e.spentUsd),e.id]}function xt(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",se(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];if(e.metrics&&Object.keys(e.metrics).length>0&&t.push(["metrics",Object.entries(e.metrics).map(([r,n])=>`${r}=${n}`).join(" \xB7 ")]),t.push(["spent",T(e.spentUsd)]),(e.status==="RUNNING"||e.status==="SCORING")&&e.liveSpentUsd!==null){let r=e.liveSpendAt?` as of ${e.liveSpendAt}`:"";t.push(["spent (live)",`at least $${e.liveSpentUsd.toFixed(4)}${r}`]);}return e.sandboxProvider&&t.push(["provider",e.sandboxProvider]),e.sandboxId&&t.push(["sandbox",e.sandboxId]),e.verifierMode&&t.push(["verifier",e.verifierMode]),e.verifierSandboxId&&t.push(["verifier sandbox",e.verifierSandboxId]),e.resolvedHarnessVersion&&t.push(["harness version",e.resolvedHarnessVersion]),e.phaseTimingsMs&&Object.keys(e.phaseTimingsMs).length>0&&t.push(["timings",Object.entries(e.phaseTimingsMs).map(([r,n])=>`${r}=${n}ms`).join(" \xB7 ")]),e.failurePhase&&t.push(["failure phase",e.failurePhase]),e.failureDetail&&t.push(["failure detail",e.failureDetail]),e.sessionRef&&t.push(["session",e.sessionRef]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function It(e){if(e===null)return "-";let t=Math.round(e*1e3)/1e3;return t>0?`+${t}`:String(t)}function Tt(e){return [e.taskKey,e.status,$(e.sourceReward),$(e.reward),It(e.rewardDelta),e.sourceTrialId]}function Te(e){let t=[["job id",e.id],["status",e.status],["source job",e.sourceJobId],["provider",e.sandboxProvider],["results",String(e.results.total)]],r=Object.entries(e.results.byStatus).filter(([,s])=>s>0).map(([s,i])=>`${s} ${i}`).join(" \xB7 ");if(r&&t.push(["by status",r]),e.filter&&(e.filter.status?.length||e.filter.taskKey)){let s=[];e.filter.status?.length&&s.push(`status=${e.filter.status.join(",")}`),e.filter.taskKey&&s.push(`task=${e.filter.taskKey}`),t.push(["filter",s.join(" \xB7 ")]);}t.push(["created",e.createdAt]);let n=h(t);if(e.results.items.length>0){n.push("");let s=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let i of e.results.items)s.push(Tt(i));n.push(...h(s)),e.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${e.id} --cursor ${e.results.nextCursor}`);}return n}function xe(e){let t=[["name",e.name],["source",e.source],["run command",e.runCommand]],r=Object.keys(e.env??{});return r.length>0&&t.push(["env",r.sort().join(", ")]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}var Ce=["e2b","daytona","modal"];function Ct(e){return Ce.filter(t=>e?.[t]!==void 0).map(t=>`${t} ${e[t].ok?"ok":"NO"}`).join(" \xB7 ")}function $(e){return e!==null?String(Math.round(e*1e3)/1e3):"-"}function Pt(e){let t=W(JSON.stringify(e.data??{}),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${t}`.trimEnd()}function Pe(e){let t=e.failures?.length?` (${e.failures.length} task failure${e.failures.length===1?"":"s"})`:"";return `${e.message}${t}`}function ne(e){let t=[["id",e.id],["status",e.status]];if(e.benchmarkName!==void 0&&t.push(["benchmark",e.benchmarkName]),e.version!==void 0&&t.push(["version",e.version]),e.taskCount!==void 0&&t.push(["tasks",String(e.taskCount)]),e.failure){t.push(["failure",Pe(e.failure)]);for(let r of e.failure.failures??[])t.push([` ${r.taskKey}`,r.error]);}return h(t)}function Et(e){let t=[];return e.taskCount!==void 0&&t.push(`tasks=${e.taskCount}`),e.failure&&t.push(W(Pe(e.failure),140)),`status ${e.status.padEnd(12)} ${t.join(" ")}`.trimEnd()}function At(e){let t={...e.data??{}},r=[];typeof t.trialId=="string"&&r.push(t.trialId),typeof t.taskKey=="string"&&r.push(t.taskKey),typeof t.status=="string"&&r.push(t.status),typeof t.reward=="number"&&r.push(`reward=${t.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[i,o]of Object.entries(t))n.has(i)||r.push(`${i}=${typeof o=="object"?JSON.stringify(o):String(o)}`);let s=W(r.join(" "),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${s}`.trimEnd()}function b(e){let t={};return typeof e.flags["api-key"]=="string"&&(t.apiKey=e.flags["api-key"]),typeof e.flags["base-url"]=="string"&&(t.baseUrl=e.flags["base-url"]),t}function G(e){return {...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{}}}function Ot(e){return e.status==="COMPLETED"?0:e.status==="FAILED"||e.status==="CANCELLED"?1:0}async function Jt(e,t){let r=ht(e),n=e.flags.json===true,s=e.flags.watch===true,i=k(b(e)),o=await i.run(r);if(!s){if(n)t.out(JSON.stringify(o));else {for(let a of D(o))t.out(a);t.out(""),t.out(`Follow it with: evolve-evals get ${o.id}`);}return 0}n?t.out(JSON.stringify({kind:"job.created",job:o})):t.out(`Job ${o.id} (${o.benchmark}) ${o.status} \u2014 watching\u2026`);let u=await i.watch(o.id,{onEvent:a=>{t.out(n?JSON.stringify({kind:"event",...a}):At(a));}});if(n)t.out(JSON.stringify({kind:"job.final",job:u}));else {t.out("");for(let a of D(u))t.out(a);}return Ot(u)}async function _t(e,t){let n=await k(b(e)).list({...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(n)),0;if(n.items.length===0)return t.out("No jobs."),0;let s=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let i of n.items)s.push(wt(i));for(let i of h(s))t.out(i);return n.nextCursor&&t.out(`
|
|
88
88
|
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function Ut(e,t){let n=await k(b(e)).get(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Lt(e,t){let r=k(b(e)),n;if(e.flags.status!==void 0&&(n=String(e.flags.status).split(",").map(o=>o.trim()).filter(Boolean),n.length===0))throw new m("--status got an empty status list");let s=await r.trials(e.positionals[0],{...n!==void 0?{status:n}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(s)),0;if(s.items.length===0)return t.out("No trials."),0;let i=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let o of s.items)i.push(St(o));for(let o of h(i))t.out(o);return t.out(`
|
|
89
89
|
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function Dt(e,t){let n=await k(b(e)).trial(e.positionals[0],e.positionals[1]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of xt(n))t.out(s);return 0}async function $t(e,t){let r=k(b(e)),n=e.flags.json===true,s=0;for await(let i of r.trialTraceEvents(e.positionals[0],e.positionals[1],{...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{}}))t.out(n?JSON.stringify(i):Pt(i)),s+=1;return !n&&s===0&&t.out("No trace events."),0}function Bt(e){let t=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of e.jobs)r.push([n.id,n.benchmark,n.status,$(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,T(n.spentUsd)]);if(t.push(...h(r)),e.taskMatrix.length>0){t.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...e.jobs.map((i,o)=>`JOB ${o+1}`)]],s=e.jobs.map(i=>i.id);for(let i of e.taskMatrix){let o=new Map(i.cells.map(u=>[u.jobId,u]));n.push([i.taskKey,i.disagreement?"!":"",...s.map(u=>{let a=o.get(u);return a?a.meanReward!==null?`${a.status} ${$(a.meanReward)}`:a.status:"-"})]);}t.push(...h(n));}return t}async function Nt(e,t){let n=await k(b(e)).compare(e.positionals);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Bt(n))t.out(s);return 0}async function jt(e,t){let n=await k(b(e)).cancel(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Ht(e,t){let n=await k(b(e)).rerunFailed(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else {for(let s of D(n))t.out(s);t.out(""),t.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function Mt(e,t){let r=k(b(e)),[n,s]=e.positionals,i;if(s!==void 0){if(e.flags.status!==void 0||e.flags.task!==void 0)throw new m("--status/--task apply to a whole-job regrade, not a single trial");i=await r.regradeTrial(n,s);}else {let o={};if(e.flags.status!==void 0){let u=String(e.flags.status).split(",").map(a=>a.trim()).filter(Boolean);if(u.length===0)throw new m("--status got an empty status list");o.status=u;}e.flags.task!==void 0&&(o.taskKey=String(e.flags.task)),i=await r.regrade(n,o);}if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of Te(i))t.out(o);t.out(""),t.out(`Follow it with: evolve-evals regrade-job ${i.id}`);}return 0}async function Vt(e,t){let n=await k(b(e)).getRegrade(e.positionals[0],G(e));if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Te(n))t.out(s);return 0}async function Kt(e,t){let r=e.flags.format;if(r!==void 0&&r!=="harbor")throw new m(`Unknown --format "${r}" (supported: harbor)`);let s=await k(b(e)).export(e.positionals[0],{to:e.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return e.flags.json===true?t.out(JSON.stringify({path:s})):t.out(`Saved ${s}`),0}async function Ft(e,t){let n=await q(b(e)).downloadPackage(e.positionals[0],{to:e.flags.to??process.cwd()});return e.flags.json===true?t.out(JSON.stringify({path:n})):t.out(`Saved ${n}`),0}function qt(e){let t=h([["name",e.name],["title",e.title??"-"],["description",e.description??"-"],["active version",e.activeVersion?.version??"-"]]);if(e.versions&&e.versions.length>0){t.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let n of e.versions)r.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);t.push(...h(r));}if(e.tasks&&e.tasks.items.length>0){t.push("",`Tasks (version ${e.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let s of e.tasks.items)r.push([s.taskKey,`${s.agentTimeoutSec}s`,`${s.verifierTimeoutSec}s`,Ct(s.providers)]);t.push(...h(r)),e.tasks.nextCursor&&t.push(`More tasks: evolve-evals benchmarks get ${e.name} --cursor ${e.tasks.nextCursor}`);let n=new Map;for(let s of e.tasks.items)for(let i of Ce){let o=s.providers?.[i];o&&!o.ok&&!n.has(`${i}:${o.reason}`)&&n.set(`${i}:${o.reason}`,`${i}: ${o.reason}`);}if(n.size>0){t.push("","Provider limitations:");for(let s of n.values())t.push(` ${s}`);}}return t}async function Gt(e,t){let r=q(b(e)),[n,s]=e.positionals;if(n===void 0){let o=await r.list(G(e));if(e.flags.json===true)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No benchmarks."),0;let u=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let a of o.items)u.push([a.name,a.activeVersion?.version??"-",a.activeVersion?.state??"-",a.activeVersion?String(a.activeVersion.taskCount):"-",a.title??"-"]);for(let a of h(u))t.out(a);for(let a of Ie(o.items))t.out(a);return 0}if(n!=="get")throw new m(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!s)throw new m('"benchmarks get" requires a <name[@version]> ref');let i=await r.get(s,G(e));if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of qt(i))t.out(o);for(let o of Ie([i]))t.out(o);}return 0}function Ie(e){let t=[];for(let r of e){if(!r.upstream?.moved)continue;let n=r.activeVersion?`@${r.activeVersion.version}`:"";t.push(`${r.name}${n} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return t}async function Wt(e,t){let[r,n]=e.positionals,s=e.flags.json===true,i=q(b(e));if(r==="status"){if(!n)throw new m('"import status" requires an <id>');let l=await i.getImport(n);if(s)t.out(JSON.stringify(l));else for(let d of ne(l))t.out(d);return 0}if(r!==void 0)throw new m(`Unknown import subcommand "${r}" (supported: status)`);let o=yt(e),u=await i.import(o);if(e.flags.watch!==true){if(s)t.out(JSON.stringify(u));else {for(let l of ne(u))t.out(l);t.out(""),t.out(`Follow it with: evolve-evals import status ${u.id}`);}return 0}s?t.out(JSON.stringify({kind:"import.created",benchmarkImport:u})):t.out(`Import ${u.id} (${o.benchmarkName}) ${u.status} \u2014 watching\u2026`);let a=await i.watchImport(u.id,{onStatus:l=>{t.out(s?JSON.stringify({kind:"import.status",benchmarkImport:l}):Et(l));}});if(s)t.out(JSON.stringify({kind:"import.final",benchmarkImport:a}));else {t.out("");for(let l of ne(a))t.out(l);}return a.status==="FAILED"?1:0}async function Yt(e,t){let r=we(b(e)),[n,s]=e.positionals,i=e.flags.json===true;if(n===void 0){let o=await r.list(G(e));if(i)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No custom harnesses."),0;let u=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let a of o.items)u.push([a.name,a.source,W(a.runCommand,60),a.updatedAt]);for(let a of h(u))t.out(a);return 0}if(n==="add"){if(s!==void 0)throw new m(`"custom-harnesses add" got unexpected argument "${s}"`);let o=await r.create(Rt(e));if(i)t.out(JSON.stringify(o));else {for(let u of xe(o))t.out(u);t.out(""),t.out(`Use it with: evolve-evals run --agent ${o.name}:<model> \u2026`);}return 0}if(n==="get"){if(!s)throw new m('"custom-harnesses get" requires a <name>');let o=await r.get(s);if(i)t.out(JSON.stringify(o));else for(let u of xe(o))t.out(u);return 0}if(n==="remove"){if(!s)throw new m('"custom-harnesses remove" requires a <name>');return await r.delete(s),i?t.out(JSON.stringify({name:s,deleted:true})):t.out(`Deleted custom harness ${s}`),0}throw new m(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function zt(e,t=vt){let r;try{r=ft(e);}catch(n){if(n instanceof m)return t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(r.command){case "help":return t.out(mt),0;case "run":return await Jt(r,t);case "list":return await _t(r,t);case "get":return await Ut(r,t);case "trials":return await Lt(r,t);case "trial":return await Dt(r,t);case "trace":return await $t(r,t);case "compare":return await Nt(r,t);case "cancel":return await jt(r,t);case "rerun-failed":return await Ht(r,t);case "regrade":return await Mt(r,t);case "regrade-job":return await Vt(r,t);case "export":return await Kt(r,t);case "benchmarks":return await Gt(r,t);case "import":return await Wt(r,t);case "download":return await Ft(r,t);case "custom-harnesses":return await Yt(r,t);default:return t.err(`Error: unknown command "${r.command}"`),2}}catch(n){return n instanceof m?(t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2):(t.err(`Error: ${n.message}`),1)}}var Qt=(()=>{let e=process.argv[1];if(!e)return false;let t=r=>{try{return (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('cli.cjs', document.baseURI).href))===url.pathToFileURL(r).href}catch{return false}};if(t(e))return true;try{return t(fs.realpathSync(e))}catch{return false}})();Qt&&zt(process.argv.slice(2)).then(e=>{process.exitCode=e;},e=>{process.stderr.write(`Error: ${e?.message??e}
|
|
90
|
-
`),process.exitCode=1;});exports.CliUsageError=m;exports.USAGE=mt;exports.buildCustomHarnessInput=Rt;exports.buildImportInput=yt;exports.buildJobInput=ht;exports.eventLine=At;exports.importStatusLine=Et;exports.parseArgs=ft;exports.parseHarnessEnv=kt;exports.parseJobAgent=bt;exports.runCli=zt;exports.traceEventLine=Pt;
|
|
90
|
+
`),process.exitCode=1;});exports.CliUsageError=m;exports.USAGE=mt;exports.buildCustomHarnessInput=Rt;exports.buildImportInput=yt;exports.buildJobInput=ht;exports.eventLine=At;exports.importStatusLine=Et;exports.parseArgs=ft;exports.parseHarnessEnv=kt;exports.parseJobAgent=bt;exports.runCli=zt;exports.traceEventLine=Pt;exports.trialDetailLines=xt;
|
package/dist/hosted/cli.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-
|
|
2
|
+
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, t as TrialDetail, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-CMNbgLpL.cjs';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* evolve-evals — CLI for Evolve hosted benchmarks & jobs.
|
|
@@ -45,6 +45,11 @@ interface CliIO {
|
|
|
45
45
|
out(line: string): void;
|
|
46
46
|
err(line: string): void;
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Full-detail rendering of one trial — evolve-evals trial. Exported for tests,
|
|
50
|
+
* like the other line renderers.
|
|
51
|
+
*/
|
|
52
|
+
declare function trialDetailLines(run: TrialDetail): string[];
|
|
48
53
|
/** One trace event line — evolve-evals trace. */
|
|
49
54
|
declare function traceEventLine(event: TrialTraceEvent): string;
|
|
50
55
|
/** Compact one-line rendering of one import status change for --watch. */
|
|
@@ -53,4 +58,4 @@ declare function importStatusLine(job: BenchmarkImport): string;
|
|
|
53
58
|
declare function eventLine(event: JobEvent): string;
|
|
54
59
|
declare function runCli(argv: string[], io?: CliIO): Promise<number>;
|
|
55
60
|
|
|
56
|
-
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };
|
|
61
|
+
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine, trialDetailLines };
|
package/dist/hosted/cli.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-
|
|
2
|
+
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, t as TrialDetail, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-CMNbgLpL.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* evolve-evals — CLI for Evolve hosted benchmarks & jobs.
|
|
@@ -45,6 +45,11 @@ interface CliIO {
|
|
|
45
45
|
out(line: string): void;
|
|
46
46
|
err(line: string): void;
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Full-detail rendering of one trial — evolve-evals trial. Exported for tests,
|
|
50
|
+
* like the other line renderers.
|
|
51
|
+
*/
|
|
52
|
+
declare function trialDetailLines(run: TrialDetail): string[];
|
|
48
53
|
/** One trace event line — evolve-evals trace. */
|
|
49
54
|
declare function traceEventLine(event: TrialTraceEvent): string;
|
|
50
55
|
/** Compact one-line rendering of one import status change for --watch. */
|
|
@@ -53,4 +58,4 @@ declare function importStatusLine(job: BenchmarkImport): string;
|
|
|
53
58
|
declare function eventLine(event: JobEvent): string;
|
|
54
59
|
declare function runCli(argv: string[], io?: CliIO): Promise<number>;
|
|
55
60
|
|
|
56
|
-
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };
|
|
61
|
+
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine, trialDetailLines };
|
package/dist/hosted/cli.js
CHANGED
|
@@ -77,9 +77,9 @@ Other options:
|
|
|
77
77
|
--format harbor Export the Harbor job-layout bundle
|
|
78
78
|
--json Machine-readable JSON output
|
|
79
79
|
--api-key <key> API key (default: $EVOLVE_API_KEY)
|
|
80
|
-
--base-url <url> API base URL (default: the Evolve dashboard API)`,l=class extends Error{constructor(e){super(e),this.name="CliUsageError";}},U={json:"boolean","api-key":"string","base-url":"string"},D={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},
|
|
80
|
+
--base-url <url> API base URL (default: the Evolve dashboard API)`,l=class extends Error{constructor(e){super(e),this.name="CliUsageError";}},U={json:"boolean","api-key":"string","base-url":"string"},D={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},x={command:"help",positionals:[],flags:{}};function L(t){if(t.length===0)throw new l("No command given");if(t[0]==="--help"||t[0]==="-h")return x;let e=t[0],s=D[e];if(!s)throw new l(`Unknown command "${e}"`);let n={},r=[];for(let o=1;o<t.length;o++){let i=t[o];if(i==="--help"||i==="-h")return x;if(!i.startsWith("--")){r.push(i);continue}let a=i.slice(2),u,d=a.indexOf("=");d!==-1&&(u=a.slice(d+1),a=a.slice(0,d));let w=s.flags[a]??U[a];if(!w)throw new l(`Unknown option --${a} for "${e}"`);if(w==="boolean"){if(u!==void 0)throw new l(`Option --${a} takes no value`);n[a]=true;continue}let p=u;if(p===void 0){let g=t[o+1];if(g===void 0||g.startsWith("--"))throw new l(`Option --${a} requires a value`);p=g,o++;}if(w==="number"){let g=Number(p);if(p.trim()===""||!Number.isFinite(g))throw new l(`Option --${a} expects a number, got "${p}"`);n[a]=g;}else if(w==="repeat"){let g=n[a]??[];g.push(p),n[a]=g;}else n[a]=p;}if(r.length<s.minPositionals)throw new l(`"${e}" requires ${s.positionalUsage??`${s.minPositionals} argument(s)`}`);if(r.length>s.maxPositionals)throw new l(`"${e}" got unexpected argument "${r[s.maxPositionals]}"`);for(let o of s.required??[])if(!(o in n))throw new l(`"${e}" requires --${o}`);return {command:e,positionals:r,flags:n}}function M(t){let e=t.indexOf(":");if(e<=0||e===t.length-1)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);let s=t.slice(0,e),n=t.slice(e+1),r=n.indexOf(":");if(r===-1)return {harness:s,model:n};let o=n.slice(0,r),i=n.slice(r+1);if(!o||!i)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);return {harness:s,model:o,harnessVersion:i}}function q(t){let e=t.flags,s;if(e.tasks!==void 0&&(s=String(e.tasks).split(",").map(n=>n.trim()).filter(Boolean),s.length===0))throw new l("--tasks got an empty task list");return {benchmark:e.benchmark,...s!==void 0?{tasks:s}:{},agents:e.agent.map(n=>{let r=M(n);return e.effort!==void 0?{...r,reasoningEffort:e.effort}:r}),...e.runs!==void 0?{runsPerTask:e.runs}:{},...e.concurrency!==void 0?{concurrency:e.concurrency}:{},...e["max-trial-spend"]!==void 0?{maxTrialSpendUsd:e["max-trial-spend"]}:{},...e.provider!==void 0?{sandboxProvider:e.provider}:{}}}function K(t){let e=t.flags,s=typeof e.dir=="string",n=typeof e.git=="string"||typeof e.ref=="string";if(s&&n)throw new l('"import" takes EITHER --dir OR --git/--ref, not both');if(s){for(let r of ["name","version"])if(typeof e[r]!="string")throw new l(`"import" requires --${r}`);return {source:{directory:e.dir},benchmarkName:e.name,version:e.version}}for(let r of ["git","ref","name","version"])if(typeof e[r]!="string"){let o=r==="git"||r==="ref"?" (or --dir for a local corpus directory)":"";throw new l(`"import" requires --${r}${o}`)}return {source:{gitUrl:e.git,ref:e.ref},benchmarkName:e.name,version:e.version}}function V(t){let e={};for(let s of t){let n=s.indexOf("=");if(n<=0)throw new l(`Invalid --env "${s}": expected KEY=VALUE`);e[s.slice(0,n)]=s.slice(n+1);}return e}function F(t,e=s=>readFileSync(s,"utf-8")){let s=t.flags,n=typeof s.dir=="string",r=typeof s["install-script"]=="string";if(n&&r)throw new l('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!r)throw new l('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let i of ["name","run"])if(typeof s[i]!="string")throw new l(`"custom-harnesses add" requires --${i}`);let o=V(s.env??[]);return {name:s.name,...n?{directory:s.dir}:{installScript:e(s["install-script"])},runCommand:s.run,...Object.keys(o).length>0?{env:o}:{}}}var H={out:t=>process.stdout.write(t+`
|
|
81
81
|
`),err:t=>process.stderr.write(t+`
|
|
82
|
-
`)};function f(t){if(t.length===0)return [];let e=[];for(let s of t)s.forEach((n,r)=>{e[r]=Math.max(e[r]??0,n.length);});return t.map(s=>s.map((n,r)=>n.padEnd(e[r])).join(" ").trimEnd())}function h(t){return typeof t=="number"?`$${t.toFixed(2)}`:"-"}function $(t){let e=`${t.harness}:${t.model}`;return t.harnessVersion?`${e}:${t.harnessVersion}`:e}function S(t,e){return t.length>e?t.slice(0,e-1)+"\u2026":t}function b(t){let e=[["id",t.id],["status",t.status],["benchmark",t.benchmark]];e.push(["agents",t.agents.map($).join(", ")]),e.push(["size",`${t.counts.agents} agent(s) x ${t.counts.tasks} task(s) = ${t.trials.total} trial(s)`]),e.push(["runs/task",String(t.runsPerTask)]),e.push(["concurrency",String(t.concurrency)]),e.push(["max spend/trial",h(t.maxTrialSpendUsd)]),e.push(["worst case",h(t.worstCaseSpendUsd)]),e.push(["provider",t.sandboxProvider]),e.push(["spent",h(t.spentUsd)]),e.push(["mean reward",t.meanReward!==null?String(t.meanReward):"-"]);let s=Object.entries(t.trials.byStatus).filter(([,n])=>n>0).map(([n,r])=>`${n} ${r}`).join(" \xB7 ");return s&&e.push(["trials",s]),t.sourceJobId&&e.push(["rerun of",t.sourceJobId]),t.idempotentReplay&&e.push(["note","idempotent replay of an existing job"]),t.failure&&e.push(["failure",`${t.failure.code}: ${t.failure.message}`]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function B(t){return [t.id,t.status,t.benchmark,String(t.trials.total),k(t.meanReward),h(t.spentUsd),t.createdAt]}function G(t){return [t.taskKey,$(t.agent),String(t.runNumber),t.status,t.reward!==null?String(t.reward):"-",h(t.spentUsd),t.id]}function _(t){let e=[["trial id",t.id],["job",t.jobId],["task",t.taskKey],["agent",$(t.agent)],["run",String(t.runNumber)],["status",t.status],["reward",t.reward!==null?String(t.reward):"-"]];
|
|
82
|
+
`)};function f(t){if(t.length===0)return [];let e=[];for(let s of t)s.forEach((n,r)=>{e[r]=Math.max(e[r]??0,n.length);});return t.map(s=>s.map((n,r)=>n.padEnd(e[r])).join(" ").trimEnd())}function h(t){return typeof t=="number"?`$${t.toFixed(2)}`:"-"}function $(t){let e=`${t.harness}:${t.model}`;return t.harnessVersion?`${e}:${t.harnessVersion}`:e}function S(t,e){return t.length>e?t.slice(0,e-1)+"\u2026":t}function b(t){let e=[["id",t.id],["status",t.status],["benchmark",t.benchmark]];e.push(["agents",t.agents.map($).join(", ")]),e.push(["size",`${t.counts.agents} agent(s) x ${t.counts.tasks} task(s) = ${t.trials.total} trial(s)`]),e.push(["runs/task",String(t.runsPerTask)]),e.push(["concurrency",String(t.concurrency)]),e.push(["max spend/trial",h(t.maxTrialSpendUsd)]),e.push(["worst case",h(t.worstCaseSpendUsd)]),e.push(["provider",t.sandboxProvider]),e.push(["spent",h(t.spentUsd)]),e.push(["mean reward",t.meanReward!==null?String(t.meanReward):"-"]);let s=Object.entries(t.trials.byStatus).filter(([,n])=>n>0).map(([n,r])=>`${n} ${r}`).join(" \xB7 ");return s&&e.push(["trials",s]),t.sourceJobId&&e.push(["rerun of",t.sourceJobId]),t.idempotentReplay&&e.push(["note","idempotent replay of an existing job"]),t.failure&&e.push(["failure",`${t.failure.code}: ${t.failure.message}`]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function B(t){return [t.id,t.status,t.benchmark,String(t.trials.total),k(t.meanReward),h(t.spentUsd),t.createdAt]}function G(t){return [t.taskKey,$(t.agent),String(t.runNumber),t.status,t.reward!==null?String(t.reward):"-",h(t.spentUsd),t.id]}function _(t){let e=[["trial id",t.id],["job",t.jobId],["task",t.taskKey],["agent",$(t.agent)],["run",String(t.runNumber)],["status",t.status],["reward",t.reward!==null?String(t.reward):"-"]];if(t.metrics&&Object.keys(t.metrics).length>0&&e.push(["metrics",Object.entries(t.metrics).map(([s,n])=>`${s}=${n}`).join(" \xB7 ")]),e.push(["spent",h(t.spentUsd)]),(t.status==="RUNNING"||t.status==="SCORING")&&t.liveSpentUsd!==null){let s=t.liveSpendAt?` as of ${t.liveSpendAt}`:"";e.push(["spent (live)",`at least $${t.liveSpentUsd.toFixed(4)}${s}`]);}return t.sandboxProvider&&e.push(["provider",t.sandboxProvider]),t.sandboxId&&e.push(["sandbox",t.sandboxId]),t.verifierMode&&e.push(["verifier",t.verifierMode]),t.verifierSandboxId&&e.push(["verifier sandbox",t.verifierSandboxId]),t.resolvedHarnessVersion&&e.push(["harness version",t.resolvedHarnessVersion]),t.phaseTimingsMs&&Object.keys(t.phaseTimingsMs).length>0&&e.push(["timings",Object.entries(t.phaseTimingsMs).map(([s,n])=>`${s}=${n}ms`).join(" \xB7 ")]),t.failurePhase&&e.push(["failure phase",t.failurePhase]),t.failureDetail&&e.push(["failure detail",t.failureDetail]),t.sessionRef&&e.push(["session",t.sessionRef]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function W(t){if(t===null)return "-";let e=Math.round(t*1e3)/1e3;return e>0?`+${e}`:String(e)}function Y(t){return [t.taskKey,t.status,k(t.sourceReward),k(t.reward),W(t.rewardDelta),t.sourceTrialId]}function P(t){let e=[["job id",t.id],["status",t.status],["source job",t.sourceJobId],["provider",t.sandboxProvider],["results",String(t.results.total)]],s=Object.entries(t.results.byStatus).filter(([,r])=>r>0).map(([r,o])=>`${r} ${o}`).join(" \xB7 ");if(s&&e.push(["by status",s]),t.filter&&(t.filter.status?.length||t.filter.taskKey)){let r=[];t.filter.status?.length&&r.push(`status=${t.filter.status.join(",")}`),t.filter.taskKey&&r.push(`task=${t.filter.taskKey}`),e.push(["filter",r.join(" \xB7 ")]);}e.push(["created",t.createdAt]);let n=f(e);if(t.results.items.length>0){n.push("");let r=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let o of t.results.items)r.push(Y(o));n.push(...f(r)),t.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${t.id} --cursor ${t.results.nextCursor}`);}return n}function E(t){let e=[["name",t.name],["source",t.source],["run command",t.runCommand]],s=Object.keys(t.env??{});return s.length>0&&e.push(["env",s.sort().join(", ")]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}var C=["e2b","daytona","modal"];function z(t){return C.filter(e=>t?.[e]!==void 0).map(e=>`${e} ${t[e].ok?"ok":"NO"}`).join(" \xB7 ")}function k(t){return t!==null?String(Math.round(t*1e3)/1e3):"-"}function Q(t){let e=S(JSON.stringify(t.data??{}),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${e}`.trimEnd()}function T(t){let e=t.failures?.length?` (${t.failures.length} task failure${t.failures.length===1?"":"s"})`:"";return `${t.message}${e}`}function I(t){let e=[["id",t.id],["status",t.status]];if(t.benchmarkName!==void 0&&e.push(["benchmark",t.benchmarkName]),t.version!==void 0&&e.push(["version",t.version]),t.taskCount!==void 0&&e.push(["tasks",String(t.taskCount)]),t.failure){e.push(["failure",T(t.failure)]);for(let s of t.failure.failures??[])e.push([` ${s.taskKey}`,s.error]);}return f(e)}function X(t){let e=[];return t.taskCount!==void 0&&e.push(`tasks=${t.taskCount}`),t.failure&&e.push(S(T(t.failure),140)),`status ${t.status.padEnd(12)} ${e.join(" ")}`.trimEnd()}function Z(t){let e={...t.data??{}},s=[];typeof e.trialId=="string"&&s.push(e.trialId),typeof e.taskKey=="string"&&s.push(e.taskKey),typeof e.status=="string"&&s.push(e.status),typeof e.reward=="number"&&s.push(`reward=${e.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[o,i]of Object.entries(e))n.has(o)||s.push(`${o}=${typeof i=="object"?JSON.stringify(i):String(i)}`);let r=S(s.join(" "),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${r}`.trimEnd()}function c(t){let e={};return typeof t.flags["api-key"]=="string"&&(e.apiKey=t.flags["api-key"]),typeof t.flags["base-url"]=="string"&&(e.baseUrl=t.flags["base-url"]),e}function v(t){return {...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{}}}function tt(t){return t.status==="COMPLETED"?0:t.status==="FAILED"||t.status==="CANCELLED"?1:0}async function et(t,e){let s=q(t),n=t.flags.json===true,r=t.flags.watch===true,o=J$1(c(t)),i=await o.run(s);if(!r){if(n)e.out(JSON.stringify(i));else {for(let u of b(i))e.out(u);e.out(""),e.out(`Follow it with: evolve-evals get ${i.id}`);}return 0}n?e.out(JSON.stringify({kind:"job.created",job:i})):e.out(`Job ${i.id} (${i.benchmark}) ${i.status} \u2014 watching\u2026`);let a=await o.watch(i.id,{onEvent:u=>{e.out(n?JSON.stringify({kind:"event",...u}):Z(u));}});if(n)e.out(JSON.stringify({kind:"job.final",job:a}));else {e.out("");for(let u of b(a))e.out(u);}return tt(a)}async function st(t,e){let n=await J$1(c(t)).list({...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(n)),0;if(n.items.length===0)return e.out("No jobs."),0;let r=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let o of n.items)r.push(B(o));for(let o of f(r))e.out(o);return n.nextCursor&&e.out(`
|
|
83
83
|
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function nt(t,e){let n=await J$1(c(t)).get(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of b(n))e.out(r);return 0}async function rt(t,e){let s=J$1(c(t)),n;if(t.flags.status!==void 0&&(n=String(t.flags.status).split(",").map(i=>i.trim()).filter(Boolean),n.length===0))throw new l("--status got an empty status list");let r=await s.trials(t.positionals[0],{...n!==void 0?{status:n}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(r)),0;if(r.items.length===0)return e.out("No trials."),0;let o=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let i of r.items)o.push(G(i));for(let i of f(o))e.out(i);return e.out(`
|
|
84
|
-
${r.items.length} trial(s) shown`),r.nextCursor&&e.out(`More: evolve-evals trials ${t.positionals[0]} --cursor ${r.nextCursor}`),0}async function it(t,e){let n=await J$1(c(t)).trial(t.positionals[0],t.positionals[1]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of _(n))e.out(r);return 0}async function ot(t,e){let s=J$1(c(t)),n=t.flags.json===true,r=0;for await(let o of s.trialTraceEvents(t.positionals[0],t.positionals[1],{...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{}}))e.out(n?JSON.stringify(o):Q(o)),r+=1;return !n&&r===0&&e.out("No trace events."),0}function at(t){let e=[],s=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of t.jobs)s.push([n.id,n.benchmark,n.status,k(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,h(n.spentUsd)]);if(e.push(...f(s)),t.taskMatrix.length>0){e.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...t.jobs.map((o,i)=>`JOB ${i+1}`)]],r=t.jobs.map(o=>o.id);for(let o of t.taskMatrix){let i=new Map(o.cells.map(a=>[a.jobId,a]));n.push([o.taskKey,o.disagreement?"!":"",...r.map(a=>{let u=i.get(a);return u?u.meanReward!==null?`${u.status} ${k(u.meanReward)}`:u.status:"-"})]);}e.push(...f(n));}return e}async function ut(t,e){let n=await J$1(c(t)).compare(t.positionals);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of at(n))e.out(r);return 0}async function lt(t,e){let n=await J$1(c(t)).cancel(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of b(n))e.out(r);return 0}async function ct(t,e){let n=await J$1(c(t)).rerunFailed(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else {for(let r of b(n))e.out(r);e.out(""),e.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function ft(t,e){let s=J$1(c(t)),[n,r]=t.positionals,o;if(r!==void 0){if(t.flags.status!==void 0||t.flags.task!==void 0)throw new l("--status/--task apply to a whole-job regrade, not a single trial");o=await s.regradeTrial(n,r);}else {let i={};if(t.flags.status!==void 0){let a=String(t.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(a.length===0)throw new l("--status got an empty status list");i.status=a;}t.flags.task!==void 0&&(i.taskKey=String(t.flags.task)),o=await s.regrade(n,i);}if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of
|
|
85
|
-
`),process.exitCode=1;});export{l as CliUsageError,J as USAGE,F as buildCustomHarnessInput,K as buildImportInput,q as buildJobInput,Z as eventLine,X as importStatusLine,L as parseArgs,V as parseHarnessEnv,M as parseJobAgent,kt as runCli,Q as traceEventLine};
|
|
84
|
+
${r.items.length} trial(s) shown`),r.nextCursor&&e.out(`More: evolve-evals trials ${t.positionals[0]} --cursor ${r.nextCursor}`),0}async function it(t,e){let n=await J$1(c(t)).trial(t.positionals[0],t.positionals[1]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of _(n))e.out(r);return 0}async function ot(t,e){let s=J$1(c(t)),n=t.flags.json===true,r=0;for await(let o of s.trialTraceEvents(t.positionals[0],t.positionals[1],{...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{}}))e.out(n?JSON.stringify(o):Q(o)),r+=1;return !n&&r===0&&e.out("No trace events."),0}function at(t){let e=[],s=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of t.jobs)s.push([n.id,n.benchmark,n.status,k(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,h(n.spentUsd)]);if(e.push(...f(s)),t.taskMatrix.length>0){e.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...t.jobs.map((o,i)=>`JOB ${i+1}`)]],r=t.jobs.map(o=>o.id);for(let o of t.taskMatrix){let i=new Map(o.cells.map(a=>[a.jobId,a]));n.push([o.taskKey,o.disagreement?"!":"",...r.map(a=>{let u=i.get(a);return u?u.meanReward!==null?`${u.status} ${k(u.meanReward)}`:u.status:"-"})]);}e.push(...f(n));}return e}async function ut(t,e){let n=await J$1(c(t)).compare(t.positionals);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of at(n))e.out(r);return 0}async function lt(t,e){let n=await J$1(c(t)).cancel(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of b(n))e.out(r);return 0}async function ct(t,e){let n=await J$1(c(t)).rerunFailed(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else {for(let r of b(n))e.out(r);e.out(""),e.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function ft(t,e){let s=J$1(c(t)),[n,r]=t.positionals,o;if(r!==void 0){if(t.flags.status!==void 0||t.flags.task!==void 0)throw new l("--status/--task apply to a whole-job regrade, not a single trial");o=await s.regradeTrial(n,r);}else {let i={};if(t.flags.status!==void 0){let a=String(t.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(a.length===0)throw new l("--status got an empty status list");i.status=a;}t.flags.task!==void 0&&(i.taskKey=String(t.flags.task)),o=await s.regrade(n,i);}if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of P(o))e.out(i);e.out(""),e.out(`Follow it with: evolve-evals regrade-job ${o.id}`);}return 0}async function dt(t,e){let n=await J$1(c(t)).getRegrade(t.positionals[0],v(t));if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of P(n))e.out(r);return 0}async function mt(t,e){let s=t.flags.format;if(s!==void 0&&s!=="harbor")throw new l(`Unknown --format "${s}" (supported: harbor)`);let r=await J$1(c(t)).export(t.positionals[0],{to:t.flags.to??process.cwd(),...s==="harbor"?{format:"harbor"}:{}});return t.flags.json===true?e.out(JSON.stringify({path:r})):e.out(`Saved ${r}`),0}async function gt(t,e){let n=await H$1(c(t)).downloadPackage(t.positionals[0],{to:t.flags.to??process.cwd()});return t.flags.json===true?e.out(JSON.stringify({path:n})):e.out(`Saved ${n}`),0}function pt(t){let e=f([["name",t.name],["title",t.title??"-"],["description",t.description??"-"],["active version",t.activeVersion?.version??"-"]]);if(t.versions&&t.versions.length>0){e.push("");let s=[["VERSION","STATE","TASKS","CREATED"]];for(let n of t.versions)s.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);e.push(...f(s));}if(t.tasks&&t.tasks.items.length>0){e.push("",`Tasks (version ${t.selectedVersion?.version??"?"}):`);let s=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let r of t.tasks.items)s.push([r.taskKey,`${r.agentTimeoutSec}s`,`${r.verifierTimeoutSec}s`,z(r.providers)]);e.push(...f(s)),t.tasks.nextCursor&&e.push(`More tasks: evolve-evals benchmarks get ${t.name} --cursor ${t.tasks.nextCursor}`);let n=new Map;for(let r of t.tasks.items)for(let o of C){let i=r.providers?.[o];i&&!i.ok&&!n.has(`${o}:${i.reason}`)&&n.set(`${o}:${i.reason}`,`${o}: ${i.reason}`);}if(n.size>0){e.push("","Provider limitations:");for(let r of n.values())e.push(` ${r}`);}}return e}async function ht(t,e){let s=H$1(c(t)),[n,r]=t.positionals;if(n===void 0){let i=await s.list(v(t));if(t.flags.json===true)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No benchmarks."),0;let a=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let u of i.items)a.push([u.name,u.activeVersion?.version??"-",u.activeVersion?.state??"-",u.activeVersion?String(u.activeVersion.taskCount):"-",u.title??"-"]);for(let u of f(a))e.out(u);for(let u of R(i.items))e.out(u);return 0}if(n!=="get")throw new l(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!r)throw new l('"benchmarks get" requires a <name[@version]> ref');let o=await s.get(r,v(t));if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of pt(o))e.out(i);for(let i of R([o]))e.out(i);}return 0}function R(t){let e=[];for(let s of t){if(!s.upstream?.moved)continue;let n=s.activeVersion?`@${s.activeVersion.version}`:"";e.push(`${s.name}${n} \xB7 upstream ${s.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${s.name} --version <new-version> --git-url <url> --ref ${s.upstream.ref}`);}return e}async function wt(t,e){let[s,n]=t.positionals,r=t.flags.json===true,o=H$1(c(t));if(s==="status"){if(!n)throw new l('"import status" requires an <id>');let d=await o.getImport(n);if(r)e.out(JSON.stringify(d));else for(let w of I(d))e.out(w);return 0}if(s!==void 0)throw new l(`Unknown import subcommand "${s}" (supported: status)`);let i=K(t),a=await o.import(i);if(t.flags.watch!==true){if(r)e.out(JSON.stringify(a));else {for(let d of I(a))e.out(d);e.out(""),e.out(`Follow it with: evolve-evals import status ${a.id}`);}return 0}r?e.out(JSON.stringify({kind:"import.created",benchmarkImport:a})):e.out(`Import ${a.id} (${i.benchmarkName}) ${a.status} \u2014 watching\u2026`);let u=await o.watchImport(a.id,{onStatus:d=>{e.out(r?JSON.stringify({kind:"import.status",benchmarkImport:d}):X(d));}});if(r)e.out(JSON.stringify({kind:"import.final",benchmarkImport:u}));else {e.out("");for(let d of I(u))e.out(d);}return u.status==="FAILED"?1:0}async function bt(t,e){let s=I$1(c(t)),[n,r]=t.positionals,o=t.flags.json===true;if(n===void 0){let i=await s.list(v(t));if(o)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No custom harnesses."),0;let a=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let u of i.items)a.push([u.name,u.source,S(u.runCommand,60),u.updatedAt]);for(let u of f(a))e.out(u);return 0}if(n==="add"){if(r!==void 0)throw new l(`"custom-harnesses add" got unexpected argument "${r}"`);let i=await s.create(F(t));if(o)e.out(JSON.stringify(i));else {for(let a of E(i))e.out(a);e.out(""),e.out(`Use it with: evolve-evals run --agent ${i.name}:<model> \u2026`);}return 0}if(n==="get"){if(!r)throw new l('"custom-harnesses get" requires a <name>');let i=await s.get(r);if(o)e.out(JSON.stringify(i));else for(let a of E(i))e.out(a);return 0}if(n==="remove"){if(!r)throw new l('"custom-harnesses remove" requires a <name>');return await s.delete(r),o?e.out(JSON.stringify({name:r,deleted:true})):e.out(`Deleted custom harness ${r}`),0}throw new l(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function kt(t,e=H){let s;try{s=L(t);}catch(n){if(n instanceof l)return e.err(`Error: ${n.message}`),e.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(s.command){case "help":return e.out(J),0;case "run":return await et(s,e);case "list":return await st(s,e);case "get":return await nt(s,e);case "trials":return await rt(s,e);case "trial":return await it(s,e);case "trace":return await ot(s,e);case "compare":return await ut(s,e);case "cancel":return await lt(s,e);case "rerun-failed":return await ct(s,e);case "regrade":return await ft(s,e);case "regrade-job":return await dt(s,e);case "export":return await mt(s,e);case "benchmarks":return await ht(s,e);case "import":return await wt(s,e);case "download":return await gt(s,e);case "custom-harnesses":return await bt(s,e);default:return e.err(`Error: unknown command "${s.command}"`),2}}catch(n){return n instanceof l?(e.err(`Error: ${n.message}`),e.err('Run "evolve-evals help" for usage.'),2):(e.err(`Error: ${n.message}`),1)}}var yt=(()=>{let t=process.argv[1];if(!t)return false;let e=s=>{try{return import.meta.url===pathToFileURL(s).href}catch{return false}};if(e(t))return true;try{return e(realpathSync(t))}catch{return false}})();yt&&kt(process.argv.slice(2)).then(t=>{process.exitCode=t;},t=>{process.stderr.write(`Error: ${t?.message??t}
|
|
85
|
+
`),process.exitCode=1;});export{l as CliUsageError,J as USAGE,F as buildCustomHarnessInput,K as buildImportInput,q as buildJobInput,Z as eventLine,X as importStatusLine,L as parseArgs,V as parseHarnessEnv,M as parseJobAgent,kt as runCli,Q as traceEventLine,_ as trialDetailLines};
|