@evolvingmachines/sdk 0.0.52-project-sable.20260726.6e95f36 → 0.0.52-project-sable.20260729.96caaab
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-ZYTUS4TY.js +6 -0
- package/dist/hosted/cli.cjs +11 -10
- package/dist/hosted/cli.d.cts +2 -2
- package/dist/hosted/cli.d.ts +2 -2
- package/dist/hosted/cli.js +8 -7
- package/dist/index.cjs +65 -65
- package/dist/index.d.cts +111 -4
- package/dist/index.d.ts +111 -4
- package/dist/index.js +20 -20
- package/dist/{types-BeJrn1lR.d.cts → types-BnWkwyuA.d.cts} +61 -6
- package/dist/{types-BeJrn1lR.d.ts → types-BnWkwyuA.d.ts} +61 -6
- package/hosted-error-codes.json +57 -0
- package/package.json +9 -6
- package/dist/chunk-IHBQMTA4.js +0 -6
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import {createWriteStream}from'fs';import {mkdir,rename,rm}from'fs/promises';import {join,basename}from'path';import {Readable}from'stream';import {pipeline}from'stream/promises';function le(e=""){let t=process.env.EVOLVE_GATEWAY_URL||"https://swarmkit-gateway-692833842999.us-central1.run.app";if(!e)return t;let n=e.startsWith("/")?e:`/${e}`;return `${t}${n}`}function de(e=""){let t=process.env.EVOLVE_DASHBOARD_URL||w;if(!e)return t;let n=e.startsWith("/")?e:`/${e}`;return `${t.replace(/\/$/,"")}${n}`}var Ue=["e2b","daytona","modal"];function W(e){return de(`/api/managed/${e}`)}function He(){return `${W("daytona")}/toolbox`}function Me(){return W("e2b")}function Ne(e){return {"browser-use":{type:"http",url:le("/browser_use/mcp"),headers:{"x-litellm-api-key":`Bearer ${e}`}}}}var M="EVOLVE_API_KEY",je="E2B_API_KEY",Ve="DAYTONA_API_KEY",$e="MODAL_TOKEN_ID",Ke="MODAL_TOKEN_SECRET",Fe="claude",Ge=36e5,qe=4,We=100,Y="/home/user";function Ye(e,t){return t?t.replace(/\/+$/,"")||"/":e?e==="root"?"/root":`/home/${e}`:Y}var w=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai",Qe="x-litellm-customer-id",Xe="x-litellm-tags",Ze="run:",et=".evolve-sdk/observability/sessions",tt=100,rt=5e3,nt=3,st=1e3;var z=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","upstream_not_watchable","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","package_not_retained","package_corrupt","package_missing","internal_error"];function N(e){return typeof e=="string"&&z.includes(e)}var A=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,n,s,o){super(s),this.name="EvolveApiError",this.status=t,this.code=n,o?.param!==void 0&&(this.param=o.param),o?.details!==void 0&&(this.details=o.details),o?.retryAfterSec!==void 0&&(this.retryAfterSec=o.retryAfterSec),o?.requestId!==void 0&&(this.requestId=o.requestId);}isKnownCode(){return N(this.code)}};async function $(e){let t=await e.text().catch(()=>""),n=e.headers?.get?.("x-request-id")??void 0,s=Number(e.headers?.get?.("retry-after")),o=Number.isFinite(s)?s:void 0;try{let d=JSON.parse(t);if(d?.error&&typeof d.error=="object"){let l=typeof d.error.code=="string"?d.error.code:"unknown_error",c=typeof d.error.message=="string"?d.error.message:e.statusText;throw new A(e.status,l,c,{param:typeof d.error.param=="string"?d.error.param:void 0,details:d.error.details&&typeof d.error.details=="object"?d.error.details:void 0,retryAfterSec:typeof d.error.retryAfterSec=="number"?d.error.retryAfterSec:o,requestId:typeof d.error.requestId=="string"?d.error.requestId:n})}}catch(d){if(d instanceof A)throw d}throw new A(e.status,"unknown_error",t||e.statusText,{retryAfterSec:o,requestId:n})}var j=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},be=1e3,fe=3e4,he=new Set(["COMPLETED","CANCELLED","FAILED"]),Re=new Set(["job.completed","job.cancelled","job.failed"]),ye=2e3,ke=new Set(["COMPLETED","FAILED"]);function K(e,t){let n=t?.apiKey||process.env[M];if(!n)throw new Error(`${e}() requires an API key. Set ${M} or pass { apiKey } in config.`);let s=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||w).replace(/\/$/,"");return {apiKey:n,baseUrl:s}}async function m(e,t,n){let s=await fetch(`${e.baseUrl}${t}`,{...n,headers:{Authorization:`Bearer ${e.apiKey}`,...n?.headers}});return s.ok||await $(s),s}function ve(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let n=e.slice(0,t).trim(),s=e.slice(t+1).trim();if(!n||!s)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:n,version:s}}function F(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function _(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function Te(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function I(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(F),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function k(e,t){let n=e??{};return {items:Array.isArray(n.items)?n.items.map(t):[],nextCursor:n.nextCursor??null,hasMore:n.hasMore===true}}function y(e,t){let n=new URLSearchParams;e?.limit!==void 0&&n.set("limit",String(e.limit)),e?.cursor&&n.set("cursor",e.cursor);for(let[o,d]of Object.entries(t??{}))d!==void 0&&n.set(o,d);let s=n.toString();return s?`?${s}`:""}function Q(e){return {id:e.id,taskKey:e.taskKey,agent:F(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function Ce(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function O(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...k(t,Ce),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function J(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function B(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function ae(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function xe(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:ae(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(F),createdAt:e.createdAt}}function Se(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:ae(e.coverage)}}function Pe(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(Se)}}function Ee(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function V(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function X(e,t){return t?.aborted?true:e?.name==="AbortError"}function D(e,t){return new Promise((n,s)=>{let o=setTimeout(()=>{t?.removeEventListener("abort",d),n();},e),d=()=>{clearTimeout(o),s(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",d,{once:true});})}function Ie(e){let t="";return {push(n){t+=n.replace(/\r\n/g,`
|
|
2
|
+
`);let s;for(;(s=t.indexOf(`
|
|
3
|
+
|
|
4
|
+
`))!==-1;){let o=t.slice(0,s);t=t.slice(s+2);let d,l,c=[];for(let r of o.split(`
|
|
5
|
+
`)){if(!r||r.startsWith(":"))continue;let a=r.indexOf(":"),u=a===-1?r:r.slice(0,a),i=a===-1?"":r.slice(a+1);i.startsWith(" ")&&(i=i.slice(1)),u==="id"?d=i:u==="event"?l=i:u==="data"&&c.push(i);}(d!==void 0||l!==void 0||c.length>0)&&e({id:d,event:l,data:c.join(`
|
|
6
|
+
`)});}}}}function P(e,t){let n,s=()=>n??=e({limit:t?.limit,cursor:t?.cursor});return {then(o,d){return s().then(o,d)},catch(o){return s().catch(o)},finally(o){return s().finally(o)},async*[Symbol.asyncIterator](){let o=t?.cursor;for(;;){let d=await e({limit:t?.limit,cursor:o});for(let l of d.items)yield l;if(!d.nextCursor)return;o=d.nextCursor;}}}}function L(e,t){let n=new FormData;for(let[s,o]of Object.entries(e))o!==void 0&&n.set(s,o);return t&&n.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),n}function Ae(e){let t,n=()=>(t||(t=(async()=>{let s=await e.next();for(;!s.done;)s=await e.next();return s.value})()),t);return {then(s,o){return n().then(s,o)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},[Symbol.asyncIterator](){return e}}}var oe="x-package-sha256",U=class extends Error{constructor(n,s){super(`downloaded bytes do not match the digest the server stated (expected ${n}, got ${s})`);this.expected=n;this.actual=s;}name="EvolveDigestMismatchError"},H=class extends Error{constructor(n,s){super(`download ended early: the server declared ${n} bytes and ${s} arrived`);this.expectedBytes=n;this.receivedBytes=s;}name="EvolveIncompleteDownloadError"};function we(e,t){let n=e.headers.get("Content-Length");if(n===null)return;let s=Number(n);if(Number.isFinite(s)&&t!==s)throw new H(s,t)}async function _e(e,t){let n=e.headers.get(oe);if(!n)return;let{createHash:s}=await import('crypto'),o=s("sha256").update(t).digest("hex");if(o!==n)throw new U(n,o)}function ie(e,t){let s=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!s)return t;let o=basename(s[1]);return o===""||o==="."||o===".."||/[/\\]/.test(o)||/[\u0000-\u001f]/.test(o)?t:o}function Oe(e){let t=K("benchmarks",e);async function n(l){let c=await m(t,`/api/benchmarks/imports/${encodeURIComponent(l)}`);return B(await c.json())}function s(l){if(!l||typeof l!="object")return null;let c=l;return {ref:c.ref,currentCommit:c.currentCommit,latestCommit:c.latestCommit??null,moved:c.moved===true,behindBy:typeof c.behindBy=="number"?c.behindBy:null,checkedAt:c.checkedAt??null,error:c.error??null}}async function o(l,c){let r=ve(l),a=y(c,{version:r.version}),i=await(await m(t,`/api/benchmarks/${encodeURIComponent(r.name)}${a}`)).json();return {name:i.name,title:i.title??null,description:i.description??null,activeVersion:i.activeVersion?_(i.activeVersion):null,versions:(i.versions||[]).map(_),selectedVersion:i.selectedVersion?_(i.selectedVersion):null,tasks:k(i.tasks,Te),upstream:s(i.upstream),createdAt:i.createdAt,updatedAt:i.updatedAt}}async function d(l){let c=await m(t,`/api/benchmarks${y(l)}`);return k(await c.json(),r=>({name:r.name,title:r.title??null,description:r.description??null,activeVersion:r.activeVersion?_(r.activeVersion):null,upstream:s(r.upstream)}))}return {list(l){return P(d,l)},get:o,async getActive(l,c){let r=await o(l,c);if(r.activeVersion===null)throw new j(l);return {name:r.name,title:r.title,description:r.description,activeVersion:r.activeVersion,version:r.activeVersion.version,tasks:r.tasks??{items:[],nextCursor:null,hasMore:false},versions:r.versions??[],createdAt:r.createdAt,updatedAt:r.updatedAt}},async import(l){let c=l.source;if(c?.directory){let{tarGzipDirectory:r}=await import('./tar-WPIXS3E6.js'),a=r(c.directory),u=await m(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:l.benchmarkName,version:l.version},{bytes:a,filename:"corpus.tar.gz"})});return B(await u.json())}if(c&&"gitUrl"in c&&c.gitUrl){let r=await m(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:l.benchmarkName,version:l.version,gitUrl:c.gitUrl,ref:c.ref})});return B(await r.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:n,async watchImport(l,c){let r=c?.pollIntervalMs??ye,a=null;for(;;){V(c?.signal);let u=await n(l);if(u.status!==a&&(a=u.status,c?.onStatus?.(u)),ke.has(u.status))return u;await D(r,c?.signal);}},downloadPackage:(async(l,c)=>{let r=await m(t,`/api/benchmarks/imports/${encodeURIComponent(l)}/package`);if(c?.stream){if(!r.body)throw new Error("Package response has no body");return r.body}if(c?.to){if(!r.body)throw new Error("Package response has no body");let u=c.to;await mkdir(u,{recursive:true});let i=join(u,ie(r,`import-${l}-corpus.tar.gz`)),{createHash:f,randomBytes:h}=await import('crypto'),R=`${i}.${h(8).toString("hex")}.part`,v=f("sha256"),T=0,p=Readable.fromWeb(r.body);p.on("data",g=>{v.update(g),T+=g.length;});try{await pipeline(p,createWriteStream(R));let g=r.headers.get("Content-Length");if(g!==null&&T!==Number(g))throw new H(Number(g),T);let C=r.headers.get(oe),x=v.digest("hex");if(C&&x!==C)throw new U(C,x);await rename(R,i);}catch(g){throw await rm(R,{force:true}).catch(()=>{}),g}return i}let a=Buffer.from(await r.arrayBuffer());return we(r,a.length),await _e(r,a),a}),listImports(l){return P(async c=>{let r=new URLSearchParams;c.limit!==void 0&&r.set("limit",String(c.limit)),c.cursor!==void 0&&r.set("cursor",c.cursor),l?.status!==void 0&&r.set("status",l.status),l?.benchmark!==void 0&&r.set("benchmark",l.benchmark);let a=r.toString()?`?${r}`:"",u=await m(t,`/api/benchmarks/imports${a}`);return k(await u.json(),B)},l)},async delete(l){await m(t,`/api/benchmarks/${encodeURIComponent(l)}`,{method:"DELETE"});}}}function Je(e){let t=K("customHarnesses",e);return {async create(n){let s=await Z("customHarnesses().create()",n),o=await m(t,"/api/custom-harnesses",{method:"POST",body:s});return J(await o.json())},list(n){return P(async s=>{let o=await m(t,`/api/custom-harnesses${y(s)}`);return k(await o.json(),J)},n)},async get(n){let s=await m(t,`/api/custom-harnesses/${encodeURIComponent(n)}`);return J(await s.json())},async delete(n){await m(t,`/api/custom-harnesses/${encodeURIComponent(n)}`,{method:"DELETE"});},async upsert(n,s){let o=await Z("customHarnesses().upsert()",{...s,name:n}),d=await m(t,`/api/custom-harnesses/${encodeURIComponent(n)}`,{method:"PUT",body:o});return J(await d.json())}}}async function Z(e,t){let n=typeof t.installScript=="string",s=typeof t.directory=="string";if(n&&s)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!n&&!s)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let o={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...n?{installScript:t.installScript}:{}};if(s){let{tarGzipDirectory:d}=await import('./tar-WPIXS3E6.js'),l=d(t.directory);return L(o,{bytes:l,filename:"source.tar.gz"})}return L(o)}function Be(e){let t=K("jobs",e);async function n(r){let a=await m(t,`/api/jobs/${encodeURIComponent(r)}`);return I(await a.json())}async function s(r){let a=await m(t,`/api/jobs${y(r)}`);return k(await a.json(),I)}async function o(r,a){let u=y(a,{status:a?.status?.length?a.status.join(","):void 0}),i=await m(t,`/api/jobs/${encodeURIComponent(r)}/trials${u}`);return k(await i.json(),Q)}async function d(r,a,u){let i=await m(t,`/api/jobs/${encodeURIComponent(r)}/trials/${encodeURIComponent(a)}/trace${y(u)}`);return k(await i.json(),Ee)}async function l(r,a){let u=a?`?format=${encodeURIComponent(a)}`:"";return m(t,`/api/jobs/${encodeURIComponent(r)}/export${u}`)}async function*c(r,a){let{onEvent:u,signal:i}=a??{},f=a?.reconnectDelayMs??be,h=a?.maxReconnectDelayMs??fe,R=null,v=false,T=false,p=f;for(;!v;){V(i);let g;try{g=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(r)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...R!==null?{"Last-Event-ID":String(R)}:{}},signal:i});}catch(b){if(X(b,i))throw b;await D(p,i),p=Math.min(p*2,h);continue}if(!g.ok){if(g.status===429||g.status>=500){await g.text().catch(()=>""),await D(p,i),p=Math.min(p*2,h);continue}await $(g);}if(!g.body)throw new Error("Event stream response has no body");let C=false,x=[],ue=Ie(b=>{let S=Number(b.id),E={seq:Number.isInteger(S)?S:-1,type:b.event||"message",data:b.data?De(b.data):{}};Number.isInteger(S)&&(R=S),C=true,Re.has(E.type)&&(v=true),x.push(E);}),G=g.body.getReader(),ce=new TextDecoder;try{for(;!v;){let{done:b,value:S}=await G.read();if(b)break;ue.push(ce.decode(S,{stream:!0}));for(let E of x)u?.(E),yield E;x.length=0;}}catch(b){if(X(b,i))throw b}finally{await G.cancel().catch(()=>{});}if(v)break;V(i);let q=await n(r);if(he.has(q.status)){if(T)return q;T=true;continue}C&&(p=f),await D(p,i),p=Math.min(p*2,h);}return n(r)}return {async run(r,a){let u=await m(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...a?.idempotencyKey?{"Idempotency-Key":a.idempotencyKey}:{}},body:JSON.stringify(r)});return I(await u.json())},get:n,list(r){return P(s,r)},trials(r,a){return P(u=>o(r,{...u,status:a?.status}),a)},async trial(r,a){let i=await(await m(t,`/api/jobs/${encodeURIComponent(r)}/trials/${encodeURIComponent(a)}`)).json();return {...Q(i),jobId:i.jobId}},trialTrace:d,async*trialTraceEvents(r,a,u){let i=u?.cursor;for(;;){let f=await d(r,a,{cursor:i,limit:u?.limit});for(let h of f.items)yield h;if(!f.nextCursor)return;i=f.nextCursor;}},watch(r,a){return Ae(c(r,a))},async cancel(r){let a=await m(t,`/api/jobs/${encodeURIComponent(r)}/cancel`,{method:"POST"});return I(await a.json())},async compare(r){let a=r.map(encodeURIComponent).join(","),i=await(await m(t,`/api/jobs/compare?ids=${a}`)).json();return {jobs:(i.jobs||[]).map(xe),taskMatrix:(i.taskMatrix||[]).map(Pe)}},async rerunFailed(r,a){let u=await m(t,`/api/jobs/${encodeURIComponent(r)}/rerun-failed`,{method:"POST",headers:a?.idempotencyKey?{"Idempotency-Key":a.idempotencyKey}:void 0});return I(await u.json())},async regrade(r,a){let u={};a?.status?.length&&(u.status=a.status),a?.taskKey!==void 0&&(u.taskKey=a.taskKey);let i=await m(t,`/api/jobs/${encodeURIComponent(r)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(u)});return O(await i.json())},async regradeTrial(r,a){let u=await m(t,`/api/jobs/${encodeURIComponent(r)}/trials/${encodeURIComponent(a)}/regrade`,{method:"POST"});return O(await u.json())},listRegrades(r){return P(async a=>{let i=await(await m(t,`/api/regrades${y(a,{jobId:r?.jobId})}`)).json();return {items:i.items.map(O),nextCursor:i.nextCursor,hasMore:i.hasMore}},r)},async getRegrade(r,a){let u=await m(t,`/api/regrades/${encodeURIComponent(r)}${y(a)}`);return O(await u.json())},export:(async(r,a)=>{let u=await l(r,a?.format);if(a?.stream){if(!u.body)throw new Error("Export response has no body");return u.body}if(a?.to){if(!u.body)throw new Error("Export response has no body");let f=a.to;await mkdir(f,{recursive:true});let h=join(f,ie(u,`job-${r}-export.json.gz`)),R=Readable.fromWeb(u.body);return await pipeline(R,createWriteStream(h)),h}let i=await u.arrayBuffer();return Buffer.from(i)})}}function De(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}function pt(e){let t,n,s;return {get benchmarks(){return t??=Oe(e)},get customHarnesses(){return n??=Je(e)},get jobs(){return s??=Be(e)},meta(){return Le(e)}}}async function Le(e){let t=(e?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||w).replace(/\/$/,""),n=await fetch(`${t}/api/meta`);return n.ok||await $(n),await n.json()}export{st as A,z as B,N as C,A as D,j as E,U as F,H as G,Oe as H,Je as I,Be as J,pt as K,Le as L,le as a,de as b,Ue as c,W as d,He as e,Me as f,Ne as g,M as h,je as i,Ve as j,$e as k,Ke as l,Fe as m,Ge as n,qe as o,We as p,Y as q,Ye as r,w as s,Qe as t,Xe as u,Ze as v,et as w,tt as x,rt as y,nt as z};
|
package/dist/hosted/cli.cjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
'use strict';var zlib=require('zlib'),fs=require('fs'),path=require('path'),url=require('url'),promises=require('fs/promises'),stream=require('stream'),promises$1=require('stream/promises');var _documentCurrentScript=typeof document!=='undefined'?document.currentScript:null;var
|
|
2
|
+
'use strict';var zlib=require('zlib'),fs=require('fs'),path=require('path'),url=require('url'),promises=require('fs/promises'),stream=require('stream'),promises$1=require('stream/promises');var _documentCurrentScript=typeof document!=='undefined'?document.currentScript:null;var Oe=Object.defineProperty;var Je=(e,t)=>()=>(e&&(t=e(e=0)),t);var _e=(e,t)=>{for(var r in t)Oe(e,r,{get:t[r],enumerable:true});};var Q={};_e(Q,{tarGzipDirectory:()=>He});function Ne(e){let t=[],r=n=>{let s=n===""?e:path.join(e,n),i=fs.readdirSync(s,{withFileTypes:true}).sort((o,u)=>o.name<u.name?-1:o.name>u.name?1:0);for(let o of i){if(o.name.startsWith("."))continue;let u=n===""?o.name:`${n}/${o.name}`,a=path.join(s,o.name),l=fs.lstatSync(a);l.isSymbolicLink()||(l.isDirectory()?r(u):l.isFile()&&t.push({rel:u,abs:a}));}};return r(""),t.sort((n,s)=>n.rel<s.rel?-1:n.rel>s.rel?1:0)}function je(e,t){let r=e,n="";if(Buffer.byteLength(r,"utf8")>100){let o=-1;for(let u=r.indexOf("/");u!==-1;u=r.indexOf("/",u+1)){let a=r.slice(0,u),l=r.slice(u+1);l.length>0&&Buffer.byteLength(l,"utf8")<=100&&Buffer.byteLength(a,"utf8")<=155&&(o=u);}if(o===-1)throw new Error(`path too long to store in a USTAR tar (max 255 bytes, split at "/"): ${e}`);n=r.slice(0,o),r=r.slice(o+1);}let s=Buffer.alloc(O);s.write(r,0,100,"utf8"),s.write("0000644\0",100,"ascii"),s.write("0000000\0",108,"ascii"),s.write("0000000\0",116,"ascii"),s.write(t.toString(8).padStart(11,"0")+"\0",124,"ascii"),s.write("00000000000\0",136,"ascii"),s.write(" ",148,"ascii"),s.write("0",156,"ascii"),s.write("ustar\0",257,"ascii"),s.write("00",263,"ascii"),n&&s.write(n,345,155,"utf8");let i=0;for(let o=0;o<O;o++)i+=s[o];return s.write(i.toString(8).padStart(6,"0")+"\0 ",148,"ascii"),s}function He(e){let t=Ne(e),r=[];for(let{rel:n,abs:s}of t){let i=fs.readFileSync(s);r.push(je(n,i.length)),r.push(i);let o=(O-i.length%O)%O;o>0&&r.push(Buffer.alloc(o));}return r.push(Buffer.alloc(O*2)),zlib.gzipSync(Buffer.concat(r),{level:9})}var O,X=Je(()=>{O=512;});var Y="EVOLVE_API_KEY";var ie=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai";var ue=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","upstream_not_watchable","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","package_not_retained","package_corrupt","package_missing","internal_error"];function z(e){return typeof e=="string"&&ue.includes(e)}var L=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,r,n,s){super(n),this.name="EvolveApiError",this.status=t,this.code=r,s?.param!==void 0&&(this.param=s.param),s?.details!==void 0&&(this.details=s.details),s?.retryAfterSec!==void 0&&(this.retryAfterSec=s.retryAfterSec),s?.requestId!==void 0&&(this.requestId=s.requestId);}isKnownCode(){return z(this.code)}};async function ye(e){let t=await e.text().catch(()=>""),r=e.headers?.get?.("x-request-id")??void 0,n=Number(e.headers?.get?.("retry-after")),s=Number.isFinite(n)?n:void 0;try{let i=JSON.parse(t);if(i?.error&&typeof i.error=="object"){let o=typeof i.error.code=="string"?i.error.code:"unknown_error",u=typeof i.error.message=="string"?i.error.message:e.statusText;throw new L(e.status,o,u,{param:typeof i.error.param=="string"?i.error.param:void 0,details:i.error.details&&typeof i.error.details=="object"?i.error.details:void 0,retryAfterSec:typeof i.error.retryAfterSec=="number"?i.error.retryAfterSec:s,requestId:typeof i.error.requestId=="string"?i.error.requestId:r})}}catch(i){if(i instanceof L)throw i}throw new L(e.status,"unknown_error",t||e.statusText,{retryAfterSec:s,requestId:r})}var Z=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},Fe=1e3,qe=3e4,Ge=new Set(["COMPLETED","CANCELLED","FAILED"]),We=new Set(["job.completed","job.cancelled","job.failed"]),Ye=2e3,ze=new Set(["COMPLETED","FAILED"]);function te(e,t){let r=t?.apiKey||process.env[Y];if(!r)throw new Error(`${e}() requires an API key. Set ${Y} or pass { apiKey } in config.`);let n=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||ie).replace(/\/$/,"");return {apiKey:r,baseUrl:n}}async function g(e,t,r){let n=await fetch(`${e.baseUrl}${t}`,{...r,headers:{Authorization:`Bearer ${e.apiKey}`,...r?.headers}});return n.ok||await ye(n),n}function Qe(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let r=e.slice(0,t).trim(),n=e.slice(t+1).trim();if(!r||!n)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:r,version:n}}function re(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function B(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function Xe(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function U(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(re),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function T(e,t){let r=e??{};return {items:Array.isArray(r.items)?r.items.map(t):[],nextCursor:r.nextCursor??null,hasMore:r.hasMore===true}}function S(e,t){let r=new URLSearchParams;e?.limit!==void 0&&r.set("limit",String(e.limit)),e?.cursor&&r.set("cursor",e.cursor);for(let[s,i]of Object.entries(t??{}))i!==void 0&&r.set(s,i);let n=r.toString();return n?`?${n}`:""}function ce(e){return {id:e.id,taskKey:e.taskKey,agent:re(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function Ze(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function N(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...T(t,Ze),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function j(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function H(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function ke(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function et(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:ke(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(re),createdAt:e.createdAt}}function tt(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:ke(e.coverage)}}function rt(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(tt)}}function nt(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function ee(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function de(e,t){return t?.aborted?true:e?.name==="AbortError"}function M(e,t){return new Promise((r,n)=>{let s=setTimeout(()=>{t?.removeEventListener("abort",i),r();},e),i=()=>{clearTimeout(s),n(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",i,{once:true});})}function st(e){let t="";return {push(r){t+=r.replace(/\r\n/g,`
|
|
3
3
|
`);let n;for(;(n=t.indexOf(`
|
|
4
4
|
|
|
5
|
-
`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let o,
|
|
6
|
-
`)){if(!
|
|
7
|
-
`)});}}}}function E(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,o){return n().then(s,o)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let o=await e({limit:t?.limit,cursor:s});for(let a of o.items)yield a;if(!o.nextCursor)return;s=o.nextCursor;}}}}function H(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function Ze(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}function Z(e){let t=Q("benchmarks",e);async function r(a){let i=await p(t,`/api/benchmarks/imports/${encodeURIComponent(a)}`);return N(await i.json())}function n(a){if(!a||typeof a!="object")return null;let i=a;return {ref:i.ref,currentCommit:i.currentCommit,latestCommit:i.latestCommit??null,moved:i.moved===true,behindBy:typeof i.behindBy=="number"?i.behindBy:null,checkedAt:i.checkedAt??null,error:i.error??null}}async function s(a,i){let u=Fe(a),l=w(i,{version:u.version}),d=await(await p(t,`/api/benchmarks/${encodeURIComponent(u.name)}${l}`)).json();return {name:d.name,title:d.title??null,description:d.description??null,activeVersion:d.activeVersion?D(d.activeVersion):null,versions:(d.versions||[]).map(D),selectedVersion:d.selectedVersion?D(d.selectedVersion):null,tasks:T(d.tasks,qe),upstream:n(d.upstream),createdAt:d.createdAt,updatedAt:d.updatedAt}}async function o(a){let i=await p(t,`/api/benchmarks${w(a)}`);return T(await i.json(),u=>({name:u.name,title:u.title??null,description:u.description??null,activeVersion:u.activeVersion?D(u.activeVersion):null,upstream:n(u.upstream)}))}return {list(a){return E(o,a)},get:s,async getActive(a,i){let u=await s(a,i);if(u.activeVersion===null)throw new Y(a);return {name:u.name,title:u.title,description:u.description,activeVersion:u.activeVersion,version:u.activeVersion.version,tasks:u.tasks??{items:[],nextCursor:null,hasMore:false},versions:u.versions??[],createdAt:u.createdAt,updatedAt:u.updatedAt}},async import(a){let i=a.source;if(i?.directory){let{tarGzipDirectory:u}=await Promise.resolve().then(()=>(W(),G)),l=u(i.directory),c=await p(t,"/api/benchmarks/imports",{method:"POST",body:H({benchmarkName:a.benchmarkName,version:a.version},{bytes:l,filename:"corpus.tar.gz"})});return N(await c.json())}if(i&&"gitUrl"in i&&i.gitUrl){let u=await p(t,"/api/benchmarks/imports",{method:"POST",body:H({benchmarkName:a.benchmarkName,version:a.version,gitUrl:i.gitUrl,ref:i.ref})});return N(await u.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(a,i){let u=i?.pollIntervalMs??Ve,l=null;for(;;){z(i?.signal);let c=await r(a);if(c.status!==l&&(l=c.status,i?.onStatus?.(c)),Ke.has(c.status))return c;await j(u,i?.signal);}},listImports(a){return E(async i=>{let u=new URLSearchParams;i.limit!==void 0&&u.set("limit",String(i.limit)),i.cursor!==void 0&&u.set("cursor",i.cursor),a?.status!==void 0&&u.set("status",a.status),a?.benchmark!==void 0&&u.set("benchmark",a.benchmark);let l=u.toString()?`?${u}`:"",c=await p(t,`/api/benchmarks/imports${l}`);return T(await c.json(),N)},a)},async delete(a){await p(t,`/api/benchmarks/${encodeURIComponent(a)}`,{method:"DELETE"});}}}function pe(e){let t=Q("customHarnesses",e);return {async create(r){let n=await de("customHarnesses().create()",r),s=await p(t,"/api/custom-harnesses",{method:"POST",body:n});return B(await s.json())},list(r){return E(async n=>{let s=await p(t,`/api/custom-harnesses${w(n)}`);return T(await s.json(),B)},r)},async get(r){let n=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return B(await n.json())},async delete(r){await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await de("customHarnesses().upsert()",{...n,name:r}),o=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return B(await o.json())}}}async function de(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:o}=await Promise.resolve().then(()=>(W(),G)),a=o(t.directory);return H(s,{bytes:a,filename:"source.tar.gz"})}return H(s)}function h(e){let t=Q("jobs",e);async function r(l){let c=await p(t,`/api/jobs/${encodeURIComponent(l)}`);return O(await c.json())}async function n(l){let c=await p(t,`/api/jobs${w(l)}`);return T(await c.json(),O)}async function s(l,c){let d=w(c,{status:c?.status?.length?c.status.join(","):void 0}),m=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials${d}`);return T(await m.json(),le)}async function o(l,c,d){let m=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/trace${w(d)}`);return T(await m.json(),Qe)}async function a(l,c){let d=c?`?format=${encodeURIComponent(c)}`:"";return p(t,`/api/jobs/${encodeURIComponent(l)}/export${d}`)}function i(l,c){let m=(l.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);return m?m[1]:`job-${c}-export.json.gz`}async function*u(l,c){let{onEvent:d,signal:m}=c??{},R=c?.reconnectDelayMs??Ne,v=c?.maxReconnectDelayMs??je,P=null,L=false,re=false,k=R;for(;!L;){z(m);let S;try{S=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(l)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...P!==null?{"Last-Event-ID":String(P)}:{}},signal:m});}catch(y){if(ce(y,m))throw y;await j(k,m),k=Math.min(k*2,v);continue}if(!S.ok){if(S.status===429||S.status>=500){await S.text().catch(()=>""),await j(k,m),k=Math.min(k*2,v);continue}await me(S);}if(!S.body)throw new Error("Event stream response has no body");let ne=false,K=[],ve=Xe(y=>{let I=Number(y.id),A={seq:Number.isInteger(I)?I:-1,type:y.event||"message",data:y.data?et(y.data):{}};Number.isInteger(I)&&(P=I),ne=true,Me.has(A.type)&&(L=true),K.push(A);}),se=S.body.getReader(),Se=new TextDecoder;try{for(;!L;){let{done:y,value:I}=await se.read();if(y)break;ve.push(Se.decode(I,{stream:!0}));for(let A of K)d?.(A),yield A;K.length=0;}}catch(y){if(ce(y,m))throw y}finally{await se.cancel().catch(()=>{});}if(L)break;z(m);let ae=await r(l);if(He.has(ae.status)){if(re)return ae;re=true;continue}ne&&(k=R),await j(k,m),k=Math.min(k*2,v);}return r(l)}return {async run(l,c){let d=await p(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...c?.idempotencyKey?{"Idempotency-Key":c.idempotencyKey}:{}},body:JSON.stringify(l)});return O(await d.json())},get:r,list(l){return E(n,l)},trials(l,c){return E(d=>s(l,{...d,status:c?.status}),c)},async trial(l,c){let m=await(await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}`)).json();return {...le(m),jobId:m.jobId}},trialTrace:o,async*trialTraceEvents(l,c,d){let m=d?.cursor;for(;;){let R=await o(l,c,{cursor:m,limit:d?.limit});for(let v of R.items)yield v;if(!R.nextCursor)return;m=R.nextCursor;}},watch(l,c){return Ze(u(l,c))},async cancel(l){let c=await p(t,`/api/jobs/${encodeURIComponent(l)}/cancel`,{method:"POST"});return O(await c.json())},async compare(l){let c=l.map(encodeURIComponent).join(","),m=await(await p(t,`/api/jobs/compare?ids=${c}`)).json();return {jobs:(m.jobs||[]).map(We),taskMatrix:(m.taskMatrix||[]).map(ze)}},async rerunFailed(l,c){let d=await p(t,`/api/jobs/${encodeURIComponent(l)}/rerun-failed`,{method:"POST",headers:c?.idempotencyKey?{"Idempotency-Key":c.idempotencyKey}:void 0});return O(await d.json())},async regrade(l,c){let d={};c?.status?.length&&(d.status=c.status),c?.taskKey!==void 0&&(d.taskKey=c.taskKey);let m=await p(t,`/api/jobs/${encodeURIComponent(l)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return $(await m.json())},async regradeTrial(l,c){let d=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/regrade`,{method:"POST"});return $(await d.json())},listRegrades(l){return E(async c=>{let m=await(await p(t,`/api/regrades${w(c,{jobId:l?.jobId})}`)).json();return {items:m.items.map($),nextCursor:m.nextCursor,hasMore:m.hasMore}},l)},async getRegrade(l,c){let d=await p(t,`/api/regrades/${encodeURIComponent(l)}${w(c)}`);return $(await d.json())},export:(async(l,c)=>{let d=await a(l,c?.format);if(c?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(c?.to){if(!d.body)throw new Error("Export response has no body");let R=c.to;await promises.mkdir(R,{recursive:true});let v=path.join(R,i(d,l)),P=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(P,fs.createWriteStream(v)),v}let m=await d.arrayBuffer();return Buffer.from(m)})}}function et(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var nt=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
5
|
+
`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let i,o,u=[];for(let a of s.split(`
|
|
6
|
+
`)){if(!a||a.startsWith(":"))continue;let l=a.indexOf(":"),d=l===-1?a:a.slice(0,l),c=l===-1?"":a.slice(l+1);c.startsWith(" ")&&(c=c.slice(1)),d==="id"?i=c:d==="event"?o=c:d==="data"&&u.push(c);}(i!==void 0||o!==void 0||u.length>0)&&e({id:i,event:o,data:u.join(`
|
|
7
|
+
`)});}}}}function J(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,i){return n().then(s,i)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let i=await e({limit:t?.limit,cursor:s});for(let o of i.items)yield o;if(!i.nextCursor)return;s=i.nextCursor;}}}}function V(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function at(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var Re="x-package-sha256",K=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},F=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function ot(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new F(n,t)}async function it(e,t){let r=e.headers.get(Re);if(!r)return;let{createHash:n}=await import('crypto'),s=n("sha256").update(t).digest("hex");if(s!==r)throw new K(r,s)}function we(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let s=path.basename(n[1]);return s===""||s==="."||s===".."||/[/\\]/.test(s)||/[\u0000-\u001f]/.test(s)?t:s}function q(e){let t=te("benchmarks",e);async function r(o){let u=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}`);return H(await u.json())}function n(o){if(!o||typeof o!="object")return null;let u=o;return {ref:u.ref,currentCommit:u.currentCommit,latestCommit:u.latestCommit??null,moved:u.moved===true,behindBy:typeof u.behindBy=="number"?u.behindBy:null,checkedAt:u.checkedAt??null,error:u.error??null}}async function s(o,u){let a=Qe(o),l=S(u,{version:a.version}),c=await(await g(t,`/api/benchmarks/${encodeURIComponent(a.name)}${l}`)).json();return {name:c.name,title:c.title??null,description:c.description??null,activeVersion:c.activeVersion?B(c.activeVersion):null,versions:(c.versions||[]).map(B),selectedVersion:c.selectedVersion?B(c.selectedVersion):null,tasks:T(c.tasks,Xe),upstream:n(c.upstream),createdAt:c.createdAt,updatedAt:c.updatedAt}}async function i(o){let u=await g(t,`/api/benchmarks${S(o)}`);return T(await u.json(),a=>({name:a.name,title:a.title??null,description:a.description??null,activeVersion:a.activeVersion?B(a.activeVersion):null,upstream:n(a.upstream)}))}return {list(o){return J(i,o)},get:s,async getActive(o,u){let a=await s(o,u);if(a.activeVersion===null)throw new Z(o);return {name:a.name,title:a.title,description:a.description,activeVersion:a.activeVersion,version:a.activeVersion.version,tasks:a.tasks??{items:[],nextCursor:null,hasMore:false},versions:a.versions??[],createdAt:a.createdAt,updatedAt:a.updatedAt}},async import(o){let u=o.source;if(u?.directory){let{tarGzipDirectory:a}=await Promise.resolve().then(()=>(X(),Q)),l=a(u.directory),d=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version},{bytes:l,filename:"corpus.tar.gz"})});return H(await d.json())}if(u&&"gitUrl"in u&&u.gitUrl){let a=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version,gitUrl:u.gitUrl,ref:u.ref})});return H(await a.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(o,u){let a=u?.pollIntervalMs??Ye,l=null;for(;;){ee(u?.signal);let d=await r(o);if(d.status!==l&&(l=d.status,u?.onStatus?.(d)),ze.has(d.status))return d;await M(a,u?.signal);}},downloadPackage:(async(o,u)=>{let a=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}/package`);if(u?.stream){if(!a.body)throw new Error("Package response has no body");return a.body}if(u?.to){if(!a.body)throw new Error("Package response has no body");let d=u.to;await promises.mkdir(d,{recursive:true});let c=path.join(d,we(a,`import-${o}-corpus.tar.gz`)),{createHash:p,randomBytes:w}=await import('crypto'),v=`${c}.${w(8).toString("hex")}.part`,C=p("sha256"),I=0,y=stream.Readable.fromWeb(a.body);y.on("data",f=>{C.update(f),I+=f.length;});try{await promises$1.pipeline(y,fs.createWriteStream(v));let f=a.headers.get("Content-Length");if(f!==null&&I!==Number(f))throw new F(Number(f),I);let P=a.headers.get(Re),E=C.digest("hex");if(P&&E!==P)throw new K(P,E);await promises.rename(v,c);}catch(f){throw await promises.rm(v,{force:true}).catch(()=>{}),f}return c}let l=Buffer.from(await a.arrayBuffer());return ot(a,l.length),await it(a,l),l}),listImports(o){return J(async u=>{let a=new URLSearchParams;u.limit!==void 0&&a.set("limit",String(u.limit)),u.cursor!==void 0&&a.set("cursor",u.cursor),o?.status!==void 0&&a.set("status",o.status),o?.benchmark!==void 0&&a.set("benchmark",o.benchmark);let l=a.toString()?`?${a}`:"",d=await g(t,`/api/benchmarks/imports${l}`);return T(await d.json(),H)},o)},async delete(o){await g(t,`/api/benchmarks/${encodeURIComponent(o)}`,{method:"DELETE"});}}}function ve(e){let t=te("customHarnesses",e);return {async create(r){let n=await me("customHarnesses().create()",r),s=await g(t,"/api/custom-harnesses",{method:"POST",body:n});return j(await s.json())},list(r){return J(async n=>{let s=await g(t,`/api/custom-harnesses${S(n)}`);return T(await s.json(),j)},r)},async get(r){let n=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return j(await n.json())},async delete(r){await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await me("customHarnesses().upsert()",{...n,name:r}),i=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return j(await i.json())}}}async function me(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:i}=await Promise.resolve().then(()=>(X(),Q)),o=i(t.directory);return V(s,{bytes:o,filename:"source.tar.gz"})}return V(s)}function k(e){let t=te("jobs",e);async function r(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}`);return U(await l.json())}async function n(a){let l=await g(t,`/api/jobs${S(a)}`);return T(await l.json(),U)}async function s(a,l){let d=S(l,{status:l?.status?.length?l.status.join(","):void 0}),c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials${d}`);return T(await c.json(),ce)}async function i(a,l,d){let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/trace${S(d)}`);return T(await c.json(),nt)}async function o(a,l){let d=l?`?format=${encodeURIComponent(l)}`:"";return g(t,`/api/jobs/${encodeURIComponent(a)}/export${d}`)}async function*u(a,l){let{onEvent:d,signal:c}=l??{},p=l?.reconnectDelayMs??Fe,w=l?.maxReconnectDelayMs??qe,v=null,C=false,I=false,y=p;for(;!C;){ee(c);let f;try{f=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(a)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...v!==null?{"Last-Event-ID":String(v)}:{}},signal:c});}catch(R){if(de(R,c))throw R;await M(y,c),y=Math.min(y*2,w);continue}if(!f.ok){if(f.status===429||f.status>=500){await f.text().catch(()=>""),await M(y,c),y=Math.min(y*2,w);continue}await ye(f);}if(!f.body)throw new Error("Event stream response has no body");let P=false,E=[],Ee=st(R=>{let A=Number(R.id),_={seq:Number.isInteger(A)?A:-1,type:R.event||"message",data:R.data?ut(R.data):{}};Number.isInteger(A)&&(v=A),P=true,We.has(_.type)&&(C=true),E.push(_);}),ae=f.body.getReader(),Ae=new TextDecoder;try{for(;!C;){let{done:R,value:A}=await ae.read();if(R)break;Ee.push(Ae.decode(A,{stream:!0}));for(let _ of E)d?.(_),yield _;E.length=0;}}catch(R){if(de(R,c))throw R}finally{await ae.cancel().catch(()=>{});}if(C)break;ee(c);let oe=await r(a);if(Ge.has(oe.status)){if(I)return oe;I=true;continue}P&&(y=p),await M(y,c),y=Math.min(y*2,w);}return r(a)}return {async run(a,l){let d=await g(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:{}},body:JSON.stringify(a)});return U(await d.json())},get:r,list(a){return J(n,a)},trials(a,l){return J(d=>s(a,{...d,status:l?.status}),l)},async trial(a,l){let c=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}`)).json();return {...ce(c),jobId:c.jobId}},trialTrace:i,async*trialTraceEvents(a,l,d){let c=d?.cursor;for(;;){let p=await i(a,l,{cursor:c,limit:d?.limit});for(let w of p.items)yield w;if(!p.nextCursor)return;c=p.nextCursor;}},watch(a,l){return at(u(a,l))},async cancel(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}/cancel`,{method:"POST"});return U(await l.json())},async compare(a){let l=a.map(encodeURIComponent).join(","),c=await(await g(t,`/api/jobs/compare?ids=${l}`)).json();return {jobs:(c.jobs||[]).map(et),taskMatrix:(c.taskMatrix||[]).map(rt)}},async rerunFailed(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/rerun-failed`,{method:"POST",headers:l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:void 0});return U(await d.json())},async regrade(a,l){let d={};l?.status?.length&&(d.status=l.status),l?.taskKey!==void 0&&(d.taskKey=l.taskKey);let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return N(await c.json())},async regradeTrial(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/regrade`,{method:"POST"});return N(await d.json())},listRegrades(a){return J(async l=>{let c=await(await g(t,`/api/regrades${S(l,{jobId:a?.jobId})}`)).json();return {items:c.items.map(N),nextCursor:c.nextCursor,hasMore:c.hasMore}},a)},async getRegrade(a,l){let d=await g(t,`/api/regrades/${encodeURIComponent(a)}${S(l)}`);return N(await d.json())},export:(async(a,l)=>{let d=await o(a,l?.format);if(l?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(l?.to){if(!d.body)throw new Error("Export response has no body");let p=l.to;await promises.mkdir(p,{recursive:true});let w=path.join(p,we(d,`job-${a}-export.json.gz`)),v=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(v,fs.createWriteStream(w)),w}let c=await d.arrayBuffer();return Buffer.from(c)})}}function ut(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var mt=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
8
8
|
|
|
9
9
|
Usage: evolve-evals <command> [options]
|
|
10
10
|
|
|
@@ -25,6 +25,7 @@ Commands:
|
|
|
25
25
|
benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)
|
|
26
26
|
import Import a benchmark from a git source or a local directory (--watch to follow)
|
|
27
27
|
import status <id> Show one import job
|
|
28
|
+
download <import-id> Download the original corpus package (owner only)
|
|
28
29
|
custom-harnesses List your registered custom harnesses
|
|
29
30
|
custom-harnesses get <name> Show one custom harness
|
|
30
31
|
custom-harnesses add Register a custom harness (install script or local directory)
|
|
@@ -77,13 +78,13 @@ Other options:
|
|
|
77
78
|
--limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection
|
|
78
79
|
(list, trials, trace, benchmarks, benchmarks get,
|
|
79
80
|
custom-harnesses, regrade-job)
|
|
80
|
-
--to <dir>
|
|
81
|
+
--to <dir> Directory to save into, for export and download (default: current dir)
|
|
81
82
|
--format harbor Export the Harbor job-layout bundle
|
|
82
83
|
--json Machine-readable JSON output
|
|
83
84
|
--api-key <key> API key (default: $EVOLVE_API_KEY)
|
|
84
|
-
--base-url <url> API base URL (default: the Evolve dashboard API)`,
|
|
85
|
+
--base-url <url> API base URL (default: the Evolve dashboard API)`,m=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},gt={json:"boolean","api-key":"string","base-url":"string"},pt={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},Se={command:"help",positionals:[],flags:{}};function ft(e){if(e.length===0)throw new m("No command given");if(e[0]==="--help"||e[0]==="-h")return Se;let t=e[0],r=pt[t];if(!r)throw new m(`Unknown command "${t}"`);let n={},s=[];for(let i=1;i<e.length;i++){let o=e[i];if(o==="--help"||o==="-h")return Se;if(!o.startsWith("--")){s.push(o);continue}let u=o.slice(2),a,l=u.indexOf("=");l!==-1&&(a=u.slice(l+1),u=u.slice(0,l));let d=r.flags[u]??gt[u];if(!d)throw new m(`Unknown option --${u} for "${t}"`);if(d==="boolean"){if(a!==void 0)throw new m(`Option --${u} takes no value`);n[u]=true;continue}let c=a;if(c===void 0){let p=e[i+1];if(p===void 0||p.startsWith("--"))throw new m(`Option --${u} requires a value`);c=p,i++;}if(d==="number"){let p=Number(c);if(c.trim()===""||!Number.isFinite(p))throw new m(`Option --${u} expects a number, got "${c}"`);n[u]=p;}else if(d==="repeat"){let p=n[u]??[];p.push(c),n[u]=p;}else n[u]=c;}if(s.length<r.minPositionals)throw new m(`"${t}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(s.length>r.maxPositionals)throw new m(`"${t}" got unexpected argument "${s[r.maxPositionals]}"`);for(let i of r.required??[])if(!(i in n))throw new m(`"${t}" requires --${i}`);return {command:t,positionals:s,flags:n}}function bt(e){let t=e.indexOf(":");if(t<=0||t===e.length-1)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);let r=e.slice(0,t),n=e.slice(t+1),s=n.indexOf(":");if(s===-1)return {harness:r,model:n};let i=n.slice(0,s),o=n.slice(s+1);if(!i||!o)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);return {harness:r,model:i,harnessVersion:o}}function ht(e){let t=e.flags,r;if(t.tasks!==void 0&&(r=String(t.tasks).split(",").map(n=>n.trim()).filter(Boolean),r.length===0))throw new m("--tasks got an empty task list");return {benchmark:t.benchmark,...r!==void 0?{tasks:r}:{},agents:t.agent.map(n=>{let s=bt(n);return t.effort!==void 0?{...s,reasoningEffort:t.effort}:s}),...t.runs!==void 0?{runsPerTask:t.runs}:{},...t.concurrency!==void 0?{concurrency:t.concurrency}:{},...t["max-trial-spend"]!==void 0?{maxTrialSpendUsd:t["max-trial-spend"]}:{},...t.provider!==void 0?{sandboxProvider:t.provider}:{}}}function yt(e){let t=e.flags,r=typeof t.dir=="string",n=typeof t.git=="string"||typeof t.ref=="string";if(r&&n)throw new m('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let s of ["name","version"])if(typeof t[s]!="string")throw new m(`"import" requires --${s}`);return {source:{directory:t.dir},benchmarkName:t.name,version:t.version}}for(let s of ["git","ref","name","version"])if(typeof t[s]!="string"){let i=s==="git"||s==="ref"?" (or --dir for a local corpus directory)":"";throw new m(`"import" requires --${s}${i}`)}return {source:{gitUrl:t.git,ref:t.ref},benchmarkName:t.name,version:t.version}}function kt(e){let t={};for(let r of e){let n=r.indexOf("=");if(n<=0)throw new m(`Invalid --env "${r}": expected KEY=VALUE`);t[r.slice(0,n)]=r.slice(n+1);}return t}function Rt(e,t=r=>fs.readFileSync(r,"utf-8")){let r=e.flags,n=typeof r.dir=="string",s=typeof r["install-script"]=="string";if(n&&s)throw new m('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!s)throw new m('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let o of ["name","run"])if(typeof r[o]!="string")throw new m(`"custom-harnesses add" requires --${o}`);let i=kt(r.env??[]);return {name:r.name,...n?{directory:r.dir}:{installScript:t(r["install-script"])},runCommand:r.run,...Object.keys(i).length>0?{env:i}:{}}}var wt={out:e=>process.stdout.write(e+`
|
|
85
86
|
`),err:e=>process.stderr.write(e+`
|
|
86
|
-
`)};function
|
|
87
|
-
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function
|
|
88
|
-
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function
|
|
89
|
-
`),process.exitCode=1;});exports.CliUsageError=
|
|
87
|
+
`)};function h(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join(" ").trimEnd())}function x(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function se(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function D(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(se).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",x(e.maxTrialSpendUsd)]),t.push(["worst case",x(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",x(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function vt(e){return [e.id,e.status,e.benchmark,String(e.trials.total),$(e.meanReward),x(e.spentUsd),e.createdAt]}function St(e){return [e.taskKey,se(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",x(e.spentUsd),e.id]}function Tt(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",se(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];return e.metrics&&Object.keys(e.metrics).length>0&&t.push(["metrics",Object.entries(e.metrics).map(([r,n])=>`${r}=${n}`).join(" \xB7 ")]),t.push(["spent",x(e.spentUsd)]),e.sandboxProvider&&t.push(["provider",e.sandboxProvider]),e.verifierMode&&t.push(["verifier",e.verifierMode]),e.resolvedHarnessVersion&&t.push(["harness version",e.resolvedHarnessVersion]),e.phaseTimingsMs&&Object.keys(e.phaseTimingsMs).length>0&&t.push(["timings",Object.entries(e.phaseTimingsMs).map(([r,n])=>`${r}=${n}ms`).join(" \xB7 ")]),e.failurePhase&&t.push(["failure phase",e.failurePhase]),e.failureDetail&&t.push(["failure detail",e.failureDetail]),e.sessionRef&&t.push(["session",e.sessionRef]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function Ct(e){if(e===null)return "-";let t=Math.round(e*1e3)/1e3;return t>0?`+${t}`:String(t)}function xt(e){return [e.taskKey,e.status,$(e.sourceReward),$(e.reward),Ct(e.rewardDelta),e.sourceTrialId]}function xe(e){let t=[["job id",e.id],["status",e.status],["source job",e.sourceJobId],["provider",e.sandboxProvider],["results",String(e.results.total)]],r=Object.entries(e.results.byStatus).filter(([,s])=>s>0).map(([s,i])=>`${s} ${i}`).join(" \xB7 ");if(r&&t.push(["by status",r]),e.filter&&(e.filter.status?.length||e.filter.taskKey)){let s=[];e.filter.status?.length&&s.push(`status=${e.filter.status.join(",")}`),e.filter.taskKey&&s.push(`task=${e.filter.taskKey}`),t.push(["filter",s.join(" \xB7 ")]);}t.push(["created",e.createdAt]);let n=h(t);if(e.results.items.length>0){n.push("");let s=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let i of e.results.items)s.push(xt(i));n.push(...h(s)),e.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${e.id} --cursor ${e.results.nextCursor}`);}return n}function Te(e){let t=[["name",e.name],["source",e.source],["run command",e.runCommand]],r=Object.keys(e.env??{});return r.length>0&&t.push(["env",r.sort().join(", ")]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}var Ie=["e2b","daytona","modal"];function It(e){return Ie.filter(t=>e?.[t]!==void 0).map(t=>`${t} ${e[t].ok?"ok":"NO"}`).join(" \xB7 ")}function $(e){return e!==null?String(Math.round(e*1e3)/1e3):"-"}function Pt(e){let t=W(JSON.stringify(e.data??{}),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${t}`.trimEnd()}function Pe(e){let t=e.failures?.length?` (${e.failures.length} task failure${e.failures.length===1?"":"s"})`:"";return `${e.message}${t}`}function ne(e){let t=[["id",e.id],["status",e.status]];if(e.benchmarkName!==void 0&&t.push(["benchmark",e.benchmarkName]),e.version!==void 0&&t.push(["version",e.version]),e.taskCount!==void 0&&t.push(["tasks",String(e.taskCount)]),e.failure){t.push(["failure",Pe(e.failure)]);for(let r of e.failure.failures??[])t.push([` ${r.taskKey}`,r.error]);}return h(t)}function Et(e){let t=[];return e.taskCount!==void 0&&t.push(`tasks=${e.taskCount}`),e.failure&&t.push(W(Pe(e.failure),140)),`status ${e.status.padEnd(12)} ${t.join(" ")}`.trimEnd()}function At(e){let t={...e.data??{}},r=[];typeof t.trialId=="string"&&r.push(t.trialId),typeof t.taskKey=="string"&&r.push(t.taskKey),typeof t.status=="string"&&r.push(t.status),typeof t.reward=="number"&&r.push(`reward=${t.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[i,o]of Object.entries(t))n.has(i)||r.push(`${i}=${typeof o=="object"?JSON.stringify(o):String(o)}`);let s=W(r.join(" "),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${s}`.trimEnd()}function b(e){let t={};return typeof e.flags["api-key"]=="string"&&(t.apiKey=e.flags["api-key"]),typeof e.flags["base-url"]=="string"&&(t.baseUrl=e.flags["base-url"]),t}function G(e){return {...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{}}}function Ot(e){return e.status==="COMPLETED"?0:e.status==="FAILED"||e.status==="CANCELLED"?1:0}async function Jt(e,t){let r=ht(e),n=e.flags.json===true,s=e.flags.watch===true,i=k(b(e)),o=await i.run(r);if(!s){if(n)t.out(JSON.stringify(o));else {for(let a of D(o))t.out(a);t.out(""),t.out(`Follow it with: evolve-evals get ${o.id}`);}return 0}n?t.out(JSON.stringify({kind:"job.created",job:o})):t.out(`Job ${o.id} (${o.benchmark}) ${o.status} \u2014 watching\u2026`);let u=await i.watch(o.id,{onEvent:a=>{t.out(n?JSON.stringify({kind:"event",...a}):At(a));}});if(n)t.out(JSON.stringify({kind:"job.final",job:u}));else {t.out("");for(let a of D(u))t.out(a);}return Ot(u)}async function _t(e,t){let n=await k(b(e)).list({...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(n)),0;if(n.items.length===0)return t.out("No jobs."),0;let s=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let i of n.items)s.push(vt(i));for(let i of h(s))t.out(i);return n.nextCursor&&t.out(`
|
|
88
|
+
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function Ut(e,t){let n=await k(b(e)).get(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Lt(e,t){let r=k(b(e)),n;if(e.flags.status!==void 0&&(n=String(e.flags.status).split(",").map(o=>o.trim()).filter(Boolean),n.length===0))throw new m("--status got an empty status list");let s=await r.trials(e.positionals[0],{...n!==void 0?{status:n}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(s)),0;if(s.items.length===0)return t.out("No trials."),0;let i=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let o of s.items)i.push(St(o));for(let o of h(i))t.out(o);return t.out(`
|
|
89
|
+
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function Dt(e,t){let n=await k(b(e)).trial(e.positionals[0],e.positionals[1]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Tt(n))t.out(s);return 0}async function $t(e,t){let r=k(b(e)),n=e.flags.json===true,s=0;for await(let i of r.trialTraceEvents(e.positionals[0],e.positionals[1],{...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{}}))t.out(n?JSON.stringify(i):Pt(i)),s+=1;return !n&&s===0&&t.out("No trace events."),0}function Bt(e){let t=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of e.jobs)r.push([n.id,n.benchmark,n.status,$(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,x(n.spentUsd)]);if(t.push(...h(r)),e.taskMatrix.length>0){t.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...e.jobs.map((i,o)=>`JOB ${o+1}`)]],s=e.jobs.map(i=>i.id);for(let i of e.taskMatrix){let o=new Map(i.cells.map(u=>[u.jobId,u]));n.push([i.taskKey,i.disagreement?"!":"",...s.map(u=>{let a=o.get(u);return a?a.meanReward!==null?`${a.status} ${$(a.meanReward)}`:a.status:"-"})]);}t.push(...h(n));}return t}async function Nt(e,t){let n=await k(b(e)).compare(e.positionals);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Bt(n))t.out(s);return 0}async function jt(e,t){let n=await k(b(e)).cancel(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Ht(e,t){let n=await k(b(e)).rerunFailed(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else {for(let s of D(n))t.out(s);t.out(""),t.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function Mt(e,t){let r=k(b(e)),[n,s]=e.positionals,i;if(s!==void 0){if(e.flags.status!==void 0||e.flags.task!==void 0)throw new m("--status/--task apply to a whole-job regrade, not a single trial");i=await r.regradeTrial(n,s);}else {let o={};if(e.flags.status!==void 0){let u=String(e.flags.status).split(",").map(a=>a.trim()).filter(Boolean);if(u.length===0)throw new m("--status got an empty status list");o.status=u;}e.flags.task!==void 0&&(o.taskKey=String(e.flags.task)),i=await r.regrade(n,o);}if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of xe(i))t.out(o);t.out(""),t.out(`Follow it with: evolve-evals regrade-job ${i.id}`);}return 0}async function Vt(e,t){let n=await k(b(e)).getRegrade(e.positionals[0],G(e));if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of xe(n))t.out(s);return 0}async function Kt(e,t){let r=e.flags.format;if(r!==void 0&&r!=="harbor")throw new m(`Unknown --format "${r}" (supported: harbor)`);let s=await k(b(e)).export(e.positionals[0],{to:e.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return e.flags.json===true?t.out(JSON.stringify({path:s})):t.out(`Saved ${s}`),0}async function Ft(e,t){let n=await q(b(e)).downloadPackage(e.positionals[0],{to:e.flags.to??process.cwd()});return e.flags.json===true?t.out(JSON.stringify({path:n})):t.out(`Saved ${n}`),0}function qt(e){let t=h([["name",e.name],["title",e.title??"-"],["description",e.description??"-"],["active version",e.activeVersion?.version??"-"]]);if(e.versions&&e.versions.length>0){t.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let n of e.versions)r.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);t.push(...h(r));}if(e.tasks&&e.tasks.items.length>0){t.push("",`Tasks (version ${e.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let s of e.tasks.items)r.push([s.taskKey,`${s.agentTimeoutSec}s`,`${s.verifierTimeoutSec}s`,It(s.providers)]);t.push(...h(r)),e.tasks.nextCursor&&t.push(`More tasks: evolve-evals benchmarks get ${e.name} --cursor ${e.tasks.nextCursor}`);let n=new Map;for(let s of e.tasks.items)for(let i of Ie){let o=s.providers?.[i];o&&!o.ok&&!n.has(`${i}:${o.reason}`)&&n.set(`${i}:${o.reason}`,`${i}: ${o.reason}`);}if(n.size>0){t.push("","Provider limitations:");for(let s of n.values())t.push(` ${s}`);}}return t}async function Gt(e,t){let r=q(b(e)),[n,s]=e.positionals;if(n===void 0){let o=await r.list(G(e));if(e.flags.json===true)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No benchmarks."),0;let u=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let a of o.items)u.push([a.name,a.activeVersion?.version??"-",a.activeVersion?.state??"-",a.activeVersion?String(a.activeVersion.taskCount):"-",a.title??"-"]);for(let a of h(u))t.out(a);for(let a of Ce(o.items))t.out(a);return 0}if(n!=="get")throw new m(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!s)throw new m('"benchmarks get" requires a <name[@version]> ref');let i=await r.get(s,G(e));if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of qt(i))t.out(o);for(let o of Ce([i]))t.out(o);}return 0}function Ce(e){let t=[];for(let r of e){if(!r.upstream?.moved)continue;let n=r.activeVersion?`@${r.activeVersion.version}`:"";t.push(`${r.name}${n} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return t}async function Wt(e,t){let[r,n]=e.positionals,s=e.flags.json===true,i=q(b(e));if(r==="status"){if(!n)throw new m('"import status" requires an <id>');let l=await i.getImport(n);if(s)t.out(JSON.stringify(l));else for(let d of ne(l))t.out(d);return 0}if(r!==void 0)throw new m(`Unknown import subcommand "${r}" (supported: status)`);let o=yt(e),u=await i.import(o);if(e.flags.watch!==true){if(s)t.out(JSON.stringify(u));else {for(let l of ne(u))t.out(l);t.out(""),t.out(`Follow it with: evolve-evals import status ${u.id}`);}return 0}s?t.out(JSON.stringify({kind:"import.created",benchmarkImport:u})):t.out(`Import ${u.id} (${o.benchmarkName}) ${u.status} \u2014 watching\u2026`);let a=await i.watchImport(u.id,{onStatus:l=>{t.out(s?JSON.stringify({kind:"import.status",benchmarkImport:l}):Et(l));}});if(s)t.out(JSON.stringify({kind:"import.final",benchmarkImport:a}));else {t.out("");for(let l of ne(a))t.out(l);}return a.status==="FAILED"?1:0}async function Yt(e,t){let r=ve(b(e)),[n,s]=e.positionals,i=e.flags.json===true;if(n===void 0){let o=await r.list(G(e));if(i)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No custom harnesses."),0;let u=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let a of o.items)u.push([a.name,a.source,W(a.runCommand,60),a.updatedAt]);for(let a of h(u))t.out(a);return 0}if(n==="add"){if(s!==void 0)throw new m(`"custom-harnesses add" got unexpected argument "${s}"`);let o=await r.create(Rt(e));if(i)t.out(JSON.stringify(o));else {for(let u of Te(o))t.out(u);t.out(""),t.out(`Use it with: evolve-evals run --agent ${o.name}:<model> \u2026`);}return 0}if(n==="get"){if(!s)throw new m('"custom-harnesses get" requires a <name>');let o=await r.get(s);if(i)t.out(JSON.stringify(o));else for(let u of Te(o))t.out(u);return 0}if(n==="remove"){if(!s)throw new m('"custom-harnesses remove" requires a <name>');return await r.delete(s),i?t.out(JSON.stringify({name:s,deleted:true})):t.out(`Deleted custom harness ${s}`),0}throw new m(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function zt(e,t=wt){let r;try{r=ft(e);}catch(n){if(n instanceof m)return t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(r.command){case "help":return t.out(mt),0;case "run":return await Jt(r,t);case "list":return await _t(r,t);case "get":return await Ut(r,t);case "trials":return await Lt(r,t);case "trial":return await Dt(r,t);case "trace":return await $t(r,t);case "compare":return await Nt(r,t);case "cancel":return await jt(r,t);case "rerun-failed":return await Ht(r,t);case "regrade":return await Mt(r,t);case "regrade-job":return await Vt(r,t);case "export":return await Kt(r,t);case "benchmarks":return await Gt(r,t);case "import":return await Wt(r,t);case "download":return await Ft(r,t);case "custom-harnesses":return await Yt(r,t);default:return t.err(`Error: unknown command "${r.command}"`),2}}catch(n){return n instanceof m?(t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2):(t.err(`Error: ${n.message}`),1)}}var Qt=(()=>{let e=process.argv[1];if(!e)return false;let t=r=>{try{return (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('cli.cjs', document.baseURI).href))===url.pathToFileURL(r).href}catch{return false}};if(t(e))return true;try{return t(fs.realpathSync(e))}catch{return false}})();Qt&&zt(process.argv.slice(2)).then(e=>{process.exitCode=e;},e=>{process.stderr.write(`Error: ${e?.message??e}
|
|
90
|
+
`),process.exitCode=1;});exports.CliUsageError=m;exports.USAGE=mt;exports.buildCustomHarnessInput=Rt;exports.buildImportInput=yt;exports.buildJobInput=ht;exports.eventLine=At;exports.importStatusLine=Et;exports.parseArgs=ft;exports.parseHarnessEnv=kt;exports.parseJobAgent=bt;exports.runCli=zt;exports.traceEventLine=Pt;
|
package/dist/hosted/cli.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-
|
|
2
|
+
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BnWkwyuA.cjs';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* evolve-evals — CLI for Evolve hosted benchmarks & jobs.
|
|
@@ -12,7 +12,7 @@ import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHa
|
|
|
12
12
|
* runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir>
|
|
15
|
+
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n download <import-id> Download the original corpus package (owner only)\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir> Directory to save into, for export and download (default: current dir)\n --format harbor Export the Harbor job-layout bundle\n --json Machine-readable JSON output\n --api-key <key> API key (default: $EVOLVE_API_KEY)\n --base-url <url> API base URL (default: the Evolve dashboard API)";
|
|
16
16
|
/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
|
|
17
17
|
declare class CliUsageError extends Error {
|
|
18
18
|
constructor(message: string);
|
package/dist/hosted/cli.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-
|
|
2
|
+
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BnWkwyuA.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* evolve-evals — CLI for Evolve hosted benchmarks & jobs.
|
|
@@ -12,7 +12,7 @@ import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHa
|
|
|
12
12
|
* runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir>
|
|
15
|
+
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n download <import-id> Download the original corpus package (owner only)\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir> Directory to save into, for export and download (default: current dir)\n --format harbor Export the Harbor job-layout bundle\n --json Machine-readable JSON output\n --api-key <key> API key (default: $EVOLVE_API_KEY)\n --base-url <url> API base URL (default: the Evolve dashboard API)";
|
|
16
16
|
/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
|
|
17
17
|
declare class CliUsageError extends Error {
|
|
18
18
|
constructor(message: string);
|