@evolvingmachines/sdk 0.0.52-project-sable.20260729.8632c49 → 0.0.52-project-sable.20260729.96caaab
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hosted/cli.cjs +6 -6
- package/dist/hosted/cli.js +6 -6
- package/package.json +6 -5
package/dist/hosted/cli.cjs
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let i,o,u=[];for(let a of s.split(`
|
|
6
6
|
`)){if(!a||a.startsWith(":"))continue;let l=a.indexOf(":"),d=l===-1?a:a.slice(0,l),c=l===-1?"":a.slice(l+1);c.startsWith(" ")&&(c=c.slice(1)),d==="id"?i=c:d==="event"?o=c:d==="data"&&u.push(c);}(i!==void 0||o!==void 0||u.length>0)&&e({id:i,event:o,data:u.join(`
|
|
7
|
-
`)});}}}}function J(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,i){return n().then(s,i)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let i=await e({limit:t?.limit,cursor:s});for(let o of i.items)yield o;if(!i.nextCursor)return;s=i.nextCursor;}}}}function V(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function at(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var Re="x-package-sha256",K=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},F=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function ot(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new F(n,t)}async function it(e,t){let r=e.headers.get(Re);if(!r)return;let{createHash:n}=await import('crypto'),s=n("sha256").update(t).digest("hex");if(s!==r)throw new K(r,s)}function we(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let s=path.basename(n[1]);return s===""||s==="."||s===".."||/[/\\]/.test(s)||/[\u0000-\u001f]/.test(s)?t:s}function q(e){let t=te("benchmarks",e);async function r(o){let u=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}`);return H(await u.json())}function n(o){if(!o||typeof o!="object")return null;let u=o;return {ref:u.ref,currentCommit:u.currentCommit,latestCommit:u.latestCommit??null,moved:u.moved===true,behindBy:typeof u.behindBy=="number"?u.behindBy:null,checkedAt:u.checkedAt??null,error:u.error??null}}async function s(o,u){let a=Qe(o),l=S(u,{version:a.version}),c=await(await g(t,`/api/benchmarks/${encodeURIComponent(a.name)}${l}`)).json();return {name:c.name,title:c.title??null,description:c.description??null,activeVersion:c.activeVersion?B(c.activeVersion):null,versions:(c.versions||[]).map(B),selectedVersion:c.selectedVersion?B(c.selectedVersion):null,tasks:T(c.tasks,Xe),upstream:n(c.upstream),createdAt:c.createdAt,updatedAt:c.updatedAt}}async function i(o){let u=await g(t,`/api/benchmarks${S(o)}`);return T(await u.json(),a=>({name:a.name,title:a.title??null,description:a.description??null,activeVersion:a.activeVersion?B(a.activeVersion):null,upstream:n(a.upstream)}))}return {list(o){return J(i,o)},get:s,async getActive(o,u){let a=await s(o,u);if(a.activeVersion===null)throw new Z(o);return {name:a.name,title:a.title,description:a.description,activeVersion:a.activeVersion,version:a.activeVersion.version,tasks:a.tasks??{items:[],nextCursor:null,hasMore:false},versions:a.versions??[],createdAt:a.createdAt,updatedAt:a.updatedAt}},async import(o){let u=o.source;if(u?.directory){let{tarGzipDirectory:a}=await Promise.resolve().then(()=>(X(),Q)),l=a(u.directory),d=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version},{bytes:l,filename:"corpus.tar.gz"})});return H(await d.json())}if(u&&"gitUrl"in u&&u.gitUrl){let a=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version,gitUrl:u.gitUrl,ref:u.ref})});return H(await a.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(o,u){let a=u?.pollIntervalMs??Ye,l=null;for(;;){ee(u?.signal);let d=await r(o);if(d.status!==l&&(l=d.status,u?.onStatus?.(d)),ze.has(d.status))return d;await M(a,u?.signal);}},downloadPackage:(async(o,u)=>{let a=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}/package`);if(u?.stream){if(!a.body)throw new Error("Package response has no body");return a.body}if(u?.to){if(!a.body)throw new Error("Package response has no body");let d=u.to;await promises.mkdir(d,{recursive:true});let c=path.join(d,we(a,`import-${o}-corpus.tar.gz`)),{createHash:p,randomBytes:w}=await import('crypto'),v=`${c}.${w(8).toString("hex")}.part`,C=p("sha256"),I=0,y=stream.Readable.fromWeb(a.body);y.on("data",f=>{C.update(f),I+=f.length;});try{await promises$1.pipeline(y,fs.createWriteStream(v));let f=a.headers.get("Content-Length");if(f!==null&&I!==Number(f))throw new F(Number(f),I);let P=a.headers.get(Re),E=C.digest("hex");if(P&&E!==P)throw new K(P,E);await promises.rename(v,c);}catch(f){throw await promises.rm(v,{force:true}).catch(()=>{}),f}return c}let l=Buffer.from(await a.arrayBuffer());return ot(a,l.length),await it(a,l),l}),listImports(o){return J(async u=>{let a=new URLSearchParams;u.limit!==void 0&&a.set("limit",String(u.limit)),u.cursor!==void 0&&a.set("cursor",u.cursor),o?.status!==void 0&&a.set("status",o.status),o?.benchmark!==void 0&&a.set("benchmark",o.benchmark);let l=a.toString()?`?${a}`:"",d=await g(t,`/api/benchmarks/imports${l}`);return T(await d.json(),H)},o)},async delete(o){await g(t,`/api/benchmarks/${encodeURIComponent(o)}`,{method:"DELETE"});}}}function ve(e){let t=te("customHarnesses",e);return {async create(r){let n=await me("customHarnesses().create()",r),s=await g(t,"/api/custom-harnesses",{method:"POST",body:n});return j(await s.json())},list(r){return J(async n=>{let s=await g(t,`/api/custom-harnesses${S(n)}`);return T(await s.json(),j)},r)},async get(r){let n=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return j(await n.json())},async delete(r){await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await me("customHarnesses().upsert()",{...n,name:r}),i=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return j(await i.json())}}}async function me(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:i}=await Promise.resolve().then(()=>(X(),Q)),o=i(t.directory);return V(s,{bytes:o,filename:"source.tar.gz"})}return V(s)}function k(e){let t=te("jobs",e);async function r(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}`);return U(await l.json())}async function n(a){let l=await g(t,`/api/jobs${S(a)}`);return T(await l.json(),U)}async function s(a,l){let d=S(l,{status:l?.status?.length?l.status.join(","):void 0}),c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials${d}`);return T(await c.json(),ce)}async function i(a,l,d){let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/trace${S(d)}`);return T(await c.json(),nt)}async function o(a,l){let d=l?`?format=${encodeURIComponent(l)}`:"";return g(t,`/api/jobs/${encodeURIComponent(a)}/export${d}`)}async function*u(a,l){let{onEvent:d,signal:c}=l??{},p=l?.reconnectDelayMs??Fe,w=l?.maxReconnectDelayMs??qe,v=null,C=false,I=false,y=p;for(;!C;){ee(c);let f;try{f=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(a)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...v!==null?{"Last-Event-ID":String(v)}:{}},signal:c});}catch(R){if(de(R,c))throw R;await M(y,c),y=Math.min(y*2,w);continue}if(!f.ok){if(f.status===429||f.status>=500){await f.text().catch(()=>""),await M(y,c),y=Math.min(y*2,w);continue}await ye(f);}if(!f.body)throw new Error("Event stream response has no body");let P=false,E=[],Ee=st(R=>{let A=Number(R.id),_={seq:Number.isInteger(A)?A:-1,type:R.event||"message",data:R.data?ut(R.data):{}};Number.isInteger(A)&&(v=A),P=true,We.has(_.type)&&(C=true),E.push(_);}),ae=f.body.getReader(),Ae=new TextDecoder;try{for(;!C;){let{done:R,value:A}=await ae.read();if(R)break;Ee.push(Ae.decode(A,{stream:!0}));for(let _ of E)d?.(_),yield _;E.length=0;}}catch(R){if(de(R,c))throw R}finally{await ae.cancel().catch(()=>{});}if(C)break;ee(c);let oe=await r(a);if(Ge.has(oe.status)){if(I)return oe;I=true;continue}P&&(y=p),await M(y,c),y=Math.min(y*2,w);}return r(a)}return {async run(a,l){let d=await g(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:{}},body:JSON.stringify(a)});return U(await d.json())},get:r,list(a){return J(n,a)},trials(a,l){return J(d=>s(a,{...d,status:l?.status}),l)},async trial(a,l){let c=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}`)).json();return {...ce(c),jobId:c.jobId}},trialTrace:i,async*trialTraceEvents(a,l,d){let c=d?.cursor;for(;;){let p=await i(a,l,{cursor:c,limit:d?.limit});for(let w of p.items)yield w;if(!p.nextCursor)return;c=p.nextCursor;}},watch(a,l){return at(u(a,l))},async cancel(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}/cancel`,{method:"POST"});return U(await l.json())},async compare(a){let l=a.map(encodeURIComponent).join(","),c=await(await g(t,`/api/jobs/compare?ids=${l}`)).json();return {jobs:(c.jobs||[]).map(et),taskMatrix:(c.taskMatrix||[]).map(rt)}},async rerunFailed(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/rerun-failed`,{method:"POST",headers:l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:void 0});return U(await d.json())},async regrade(a,l){let d={};l?.status?.length&&(d.status=l.status),l?.taskKey!==void 0&&(d.taskKey=l.taskKey);let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return N(await c.json())},async regradeTrial(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/regrade`,{method:"POST"});return N(await d.json())},listRegrades(a){return J(async l=>{let c=await(await g(t,`/api/regrades${S(l,{jobId:a?.jobId})}`)).json();return {items:c.items.map(N),nextCursor:c.nextCursor,hasMore:c.hasMore}},a)},async getRegrade(a,l){let d=await g(t,`/api/regrades/${encodeURIComponent(a)}${S(l)}`);return N(await d.json())},export:(async(a,l)=>{let d=await o(a,l?.format);if(l?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(l?.to){if(!d.body)throw new Error("Export response has no body");let p=l.to;await promises.mkdir(p,{recursive:true});let w=path.join(p,we(d,`job-${a}-export.json.gz`)),v=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(v,fs.createWriteStream(w)),w}let c=await d.arrayBuffer();return Buffer.from(c)})}}function ut(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var dt=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
7
|
+
`)});}}}}function J(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,i){return n().then(s,i)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let i=await e({limit:t?.limit,cursor:s});for(let o of i.items)yield o;if(!i.nextCursor)return;s=i.nextCursor;}}}}function V(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function at(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var Re="x-package-sha256",K=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},F=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function ot(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new F(n,t)}async function it(e,t){let r=e.headers.get(Re);if(!r)return;let{createHash:n}=await import('crypto'),s=n("sha256").update(t).digest("hex");if(s!==r)throw new K(r,s)}function we(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let s=path.basename(n[1]);return s===""||s==="."||s===".."||/[/\\]/.test(s)||/[\u0000-\u001f]/.test(s)?t:s}function q(e){let t=te("benchmarks",e);async function r(o){let u=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}`);return H(await u.json())}function n(o){if(!o||typeof o!="object")return null;let u=o;return {ref:u.ref,currentCommit:u.currentCommit,latestCommit:u.latestCommit??null,moved:u.moved===true,behindBy:typeof u.behindBy=="number"?u.behindBy:null,checkedAt:u.checkedAt??null,error:u.error??null}}async function s(o,u){let a=Qe(o),l=S(u,{version:a.version}),c=await(await g(t,`/api/benchmarks/${encodeURIComponent(a.name)}${l}`)).json();return {name:c.name,title:c.title??null,description:c.description??null,activeVersion:c.activeVersion?B(c.activeVersion):null,versions:(c.versions||[]).map(B),selectedVersion:c.selectedVersion?B(c.selectedVersion):null,tasks:T(c.tasks,Xe),upstream:n(c.upstream),createdAt:c.createdAt,updatedAt:c.updatedAt}}async function i(o){let u=await g(t,`/api/benchmarks${S(o)}`);return T(await u.json(),a=>({name:a.name,title:a.title??null,description:a.description??null,activeVersion:a.activeVersion?B(a.activeVersion):null,upstream:n(a.upstream)}))}return {list(o){return J(i,o)},get:s,async getActive(o,u){let a=await s(o,u);if(a.activeVersion===null)throw new Z(o);return {name:a.name,title:a.title,description:a.description,activeVersion:a.activeVersion,version:a.activeVersion.version,tasks:a.tasks??{items:[],nextCursor:null,hasMore:false},versions:a.versions??[],createdAt:a.createdAt,updatedAt:a.updatedAt}},async import(o){let u=o.source;if(u?.directory){let{tarGzipDirectory:a}=await Promise.resolve().then(()=>(X(),Q)),l=a(u.directory),d=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version},{bytes:l,filename:"corpus.tar.gz"})});return H(await d.json())}if(u&&"gitUrl"in u&&u.gitUrl){let a=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version,gitUrl:u.gitUrl,ref:u.ref})});return H(await a.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(o,u){let a=u?.pollIntervalMs??Ye,l=null;for(;;){ee(u?.signal);let d=await r(o);if(d.status!==l&&(l=d.status,u?.onStatus?.(d)),ze.has(d.status))return d;await M(a,u?.signal);}},downloadPackage:(async(o,u)=>{let a=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}/package`);if(u?.stream){if(!a.body)throw new Error("Package response has no body");return a.body}if(u?.to){if(!a.body)throw new Error("Package response has no body");let d=u.to;await promises.mkdir(d,{recursive:true});let c=path.join(d,we(a,`import-${o}-corpus.tar.gz`)),{createHash:p,randomBytes:w}=await import('crypto'),v=`${c}.${w(8).toString("hex")}.part`,C=p("sha256"),I=0,y=stream.Readable.fromWeb(a.body);y.on("data",f=>{C.update(f),I+=f.length;});try{await promises$1.pipeline(y,fs.createWriteStream(v));let f=a.headers.get("Content-Length");if(f!==null&&I!==Number(f))throw new F(Number(f),I);let P=a.headers.get(Re),E=C.digest("hex");if(P&&E!==P)throw new K(P,E);await promises.rename(v,c);}catch(f){throw await promises.rm(v,{force:true}).catch(()=>{}),f}return c}let l=Buffer.from(await a.arrayBuffer());return ot(a,l.length),await it(a,l),l}),listImports(o){return J(async u=>{let a=new URLSearchParams;u.limit!==void 0&&a.set("limit",String(u.limit)),u.cursor!==void 0&&a.set("cursor",u.cursor),o?.status!==void 0&&a.set("status",o.status),o?.benchmark!==void 0&&a.set("benchmark",o.benchmark);let l=a.toString()?`?${a}`:"",d=await g(t,`/api/benchmarks/imports${l}`);return T(await d.json(),H)},o)},async delete(o){await g(t,`/api/benchmarks/${encodeURIComponent(o)}`,{method:"DELETE"});}}}function ve(e){let t=te("customHarnesses",e);return {async create(r){let n=await me("customHarnesses().create()",r),s=await g(t,"/api/custom-harnesses",{method:"POST",body:n});return j(await s.json())},list(r){return J(async n=>{let s=await g(t,`/api/custom-harnesses${S(n)}`);return T(await s.json(),j)},r)},async get(r){let n=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return j(await n.json())},async delete(r){await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await me("customHarnesses().upsert()",{...n,name:r}),i=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return j(await i.json())}}}async function me(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:i}=await Promise.resolve().then(()=>(X(),Q)),o=i(t.directory);return V(s,{bytes:o,filename:"source.tar.gz"})}return V(s)}function k(e){let t=te("jobs",e);async function r(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}`);return U(await l.json())}async function n(a){let l=await g(t,`/api/jobs${S(a)}`);return T(await l.json(),U)}async function s(a,l){let d=S(l,{status:l?.status?.length?l.status.join(","):void 0}),c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials${d}`);return T(await c.json(),ce)}async function i(a,l,d){let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/trace${S(d)}`);return T(await c.json(),nt)}async function o(a,l){let d=l?`?format=${encodeURIComponent(l)}`:"";return g(t,`/api/jobs/${encodeURIComponent(a)}/export${d}`)}async function*u(a,l){let{onEvent:d,signal:c}=l??{},p=l?.reconnectDelayMs??Fe,w=l?.maxReconnectDelayMs??qe,v=null,C=false,I=false,y=p;for(;!C;){ee(c);let f;try{f=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(a)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...v!==null?{"Last-Event-ID":String(v)}:{}},signal:c});}catch(R){if(de(R,c))throw R;await M(y,c),y=Math.min(y*2,w);continue}if(!f.ok){if(f.status===429||f.status>=500){await f.text().catch(()=>""),await M(y,c),y=Math.min(y*2,w);continue}await ye(f);}if(!f.body)throw new Error("Event stream response has no body");let P=false,E=[],Ee=st(R=>{let A=Number(R.id),_={seq:Number.isInteger(A)?A:-1,type:R.event||"message",data:R.data?ut(R.data):{}};Number.isInteger(A)&&(v=A),P=true,We.has(_.type)&&(C=true),E.push(_);}),ae=f.body.getReader(),Ae=new TextDecoder;try{for(;!C;){let{done:R,value:A}=await ae.read();if(R)break;Ee.push(Ae.decode(A,{stream:!0}));for(let _ of E)d?.(_),yield _;E.length=0;}}catch(R){if(de(R,c))throw R}finally{await ae.cancel().catch(()=>{});}if(C)break;ee(c);let oe=await r(a);if(Ge.has(oe.status)){if(I)return oe;I=true;continue}P&&(y=p),await M(y,c),y=Math.min(y*2,w);}return r(a)}return {async run(a,l){let d=await g(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:{}},body:JSON.stringify(a)});return U(await d.json())},get:r,list(a){return J(n,a)},trials(a,l){return J(d=>s(a,{...d,status:l?.status}),l)},async trial(a,l){let c=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}`)).json();return {...ce(c),jobId:c.jobId}},trialTrace:i,async*trialTraceEvents(a,l,d){let c=d?.cursor;for(;;){let p=await i(a,l,{cursor:c,limit:d?.limit});for(let w of p.items)yield w;if(!p.nextCursor)return;c=p.nextCursor;}},watch(a,l){return at(u(a,l))},async cancel(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}/cancel`,{method:"POST"});return U(await l.json())},async compare(a){let l=a.map(encodeURIComponent).join(","),c=await(await g(t,`/api/jobs/compare?ids=${l}`)).json();return {jobs:(c.jobs||[]).map(et),taskMatrix:(c.taskMatrix||[]).map(rt)}},async rerunFailed(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/rerun-failed`,{method:"POST",headers:l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:void 0});return U(await d.json())},async regrade(a,l){let d={};l?.status?.length&&(d.status=l.status),l?.taskKey!==void 0&&(d.taskKey=l.taskKey);let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return N(await c.json())},async regradeTrial(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/regrade`,{method:"POST"});return N(await d.json())},listRegrades(a){return J(async l=>{let c=await(await g(t,`/api/regrades${S(l,{jobId:a?.jobId})}`)).json();return {items:c.items.map(N),nextCursor:c.nextCursor,hasMore:c.hasMore}},a)},async getRegrade(a,l){let d=await g(t,`/api/regrades/${encodeURIComponent(a)}${S(l)}`);return N(await d.json())},export:(async(a,l)=>{let d=await o(a,l?.format);if(l?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(l?.to){if(!d.body)throw new Error("Export response has no body");let p=l.to;await promises.mkdir(p,{recursive:true});let w=path.join(p,we(d,`job-${a}-export.json.gz`)),v=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(v,fs.createWriteStream(w)),w}let c=await d.arrayBuffer();return Buffer.from(c)})}}function ut(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var mt=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
8
8
|
|
|
9
9
|
Usage: evolve-evals <command> [options]
|
|
10
10
|
|
|
@@ -82,9 +82,9 @@ Other options:
|
|
|
82
82
|
--format harbor Export the Harbor job-layout bundle
|
|
83
83
|
--json Machine-readable JSON output
|
|
84
84
|
--api-key <key> API key (default: $EVOLVE_API_KEY)
|
|
85
|
-
--base-url <url> API base URL (default: the Evolve dashboard API)`,m=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},
|
|
85
|
+
--base-url <url> API base URL (default: the Evolve dashboard API)`,m=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},gt={json:"boolean","api-key":"string","base-url":"string"},pt={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},Se={command:"help",positionals:[],flags:{}};function ft(e){if(e.length===0)throw new m("No command given");if(e[0]==="--help"||e[0]==="-h")return Se;let t=e[0],r=pt[t];if(!r)throw new m(`Unknown command "${t}"`);let n={},s=[];for(let i=1;i<e.length;i++){let o=e[i];if(o==="--help"||o==="-h")return Se;if(!o.startsWith("--")){s.push(o);continue}let u=o.slice(2),a,l=u.indexOf("=");l!==-1&&(a=u.slice(l+1),u=u.slice(0,l));let d=r.flags[u]??gt[u];if(!d)throw new m(`Unknown option --${u} for "${t}"`);if(d==="boolean"){if(a!==void 0)throw new m(`Option --${u} takes no value`);n[u]=true;continue}let c=a;if(c===void 0){let p=e[i+1];if(p===void 0||p.startsWith("--"))throw new m(`Option --${u} requires a value`);c=p,i++;}if(d==="number"){let p=Number(c);if(c.trim()===""||!Number.isFinite(p))throw new m(`Option --${u} expects a number, got "${c}"`);n[u]=p;}else if(d==="repeat"){let p=n[u]??[];p.push(c),n[u]=p;}else n[u]=c;}if(s.length<r.minPositionals)throw new m(`"${t}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(s.length>r.maxPositionals)throw new m(`"${t}" got unexpected argument "${s[r.maxPositionals]}"`);for(let i of r.required??[])if(!(i in n))throw new m(`"${t}" requires --${i}`);return {command:t,positionals:s,flags:n}}function bt(e){let t=e.indexOf(":");if(t<=0||t===e.length-1)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);let r=e.slice(0,t),n=e.slice(t+1),s=n.indexOf(":");if(s===-1)return {harness:r,model:n};let i=n.slice(0,s),o=n.slice(s+1);if(!i||!o)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);return {harness:r,model:i,harnessVersion:o}}function ht(e){let t=e.flags,r;if(t.tasks!==void 0&&(r=String(t.tasks).split(",").map(n=>n.trim()).filter(Boolean),r.length===0))throw new m("--tasks got an empty task list");return {benchmark:t.benchmark,...r!==void 0?{tasks:r}:{},agents:t.agent.map(n=>{let s=bt(n);return t.effort!==void 0?{...s,reasoningEffort:t.effort}:s}),...t.runs!==void 0?{runsPerTask:t.runs}:{},...t.concurrency!==void 0?{concurrency:t.concurrency}:{},...t["max-trial-spend"]!==void 0?{maxTrialSpendUsd:t["max-trial-spend"]}:{},...t.provider!==void 0?{sandboxProvider:t.provider}:{}}}function yt(e){let t=e.flags,r=typeof t.dir=="string",n=typeof t.git=="string"||typeof t.ref=="string";if(r&&n)throw new m('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let s of ["name","version"])if(typeof t[s]!="string")throw new m(`"import" requires --${s}`);return {source:{directory:t.dir},benchmarkName:t.name,version:t.version}}for(let s of ["git","ref","name","version"])if(typeof t[s]!="string"){let i=s==="git"||s==="ref"?" (or --dir for a local corpus directory)":"";throw new m(`"import" requires --${s}${i}`)}return {source:{gitUrl:t.git,ref:t.ref},benchmarkName:t.name,version:t.version}}function kt(e){let t={};for(let r of e){let n=r.indexOf("=");if(n<=0)throw new m(`Invalid --env "${r}": expected KEY=VALUE`);t[r.slice(0,n)]=r.slice(n+1);}return t}function Rt(e,t=r=>fs.readFileSync(r,"utf-8")){let r=e.flags,n=typeof r.dir=="string",s=typeof r["install-script"]=="string";if(n&&s)throw new m('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!s)throw new m('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let o of ["name","run"])if(typeof r[o]!="string")throw new m(`"custom-harnesses add" requires --${o}`);let i=kt(r.env??[]);return {name:r.name,...n?{directory:r.dir}:{installScript:t(r["install-script"])},runCommand:r.run,...Object.keys(i).length>0?{env:i}:{}}}var wt={out:e=>process.stdout.write(e+`
|
|
86
86
|
`),err:e=>process.stderr.write(e+`
|
|
87
|
-
`)};function h(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join(" ").trimEnd())}function x(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function se(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function D(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(se).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",x(e.maxTrialSpendUsd)]),t.push(["worst case",x(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",x(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function
|
|
88
|
-
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function
|
|
89
|
-
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function
|
|
90
|
-
`),process.exitCode=1;});exports.CliUsageError=m;exports.USAGE=
|
|
87
|
+
`)};function h(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join(" ").trimEnd())}function x(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function se(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function D(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(se).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",x(e.maxTrialSpendUsd)]),t.push(["worst case",x(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",x(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function vt(e){return [e.id,e.status,e.benchmark,String(e.trials.total),$(e.meanReward),x(e.spentUsd),e.createdAt]}function St(e){return [e.taskKey,se(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",x(e.spentUsd),e.id]}function Tt(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",se(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];return e.metrics&&Object.keys(e.metrics).length>0&&t.push(["metrics",Object.entries(e.metrics).map(([r,n])=>`${r}=${n}`).join(" \xB7 ")]),t.push(["spent",x(e.spentUsd)]),e.sandboxProvider&&t.push(["provider",e.sandboxProvider]),e.verifierMode&&t.push(["verifier",e.verifierMode]),e.resolvedHarnessVersion&&t.push(["harness version",e.resolvedHarnessVersion]),e.phaseTimingsMs&&Object.keys(e.phaseTimingsMs).length>0&&t.push(["timings",Object.entries(e.phaseTimingsMs).map(([r,n])=>`${r}=${n}ms`).join(" \xB7 ")]),e.failurePhase&&t.push(["failure phase",e.failurePhase]),e.failureDetail&&t.push(["failure detail",e.failureDetail]),e.sessionRef&&t.push(["session",e.sessionRef]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function Ct(e){if(e===null)return "-";let t=Math.round(e*1e3)/1e3;return t>0?`+${t}`:String(t)}function xt(e){return [e.taskKey,e.status,$(e.sourceReward),$(e.reward),Ct(e.rewardDelta),e.sourceTrialId]}function xe(e){let t=[["job id",e.id],["status",e.status],["source job",e.sourceJobId],["provider",e.sandboxProvider],["results",String(e.results.total)]],r=Object.entries(e.results.byStatus).filter(([,s])=>s>0).map(([s,i])=>`${s} ${i}`).join(" \xB7 ");if(r&&t.push(["by status",r]),e.filter&&(e.filter.status?.length||e.filter.taskKey)){let s=[];e.filter.status?.length&&s.push(`status=${e.filter.status.join(",")}`),e.filter.taskKey&&s.push(`task=${e.filter.taskKey}`),t.push(["filter",s.join(" \xB7 ")]);}t.push(["created",e.createdAt]);let n=h(t);if(e.results.items.length>0){n.push("");let s=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let i of e.results.items)s.push(xt(i));n.push(...h(s)),e.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${e.id} --cursor ${e.results.nextCursor}`);}return n}function Te(e){let t=[["name",e.name],["source",e.source],["run command",e.runCommand]],r=Object.keys(e.env??{});return r.length>0&&t.push(["env",r.sort().join(", ")]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}var Ie=["e2b","daytona","modal"];function It(e){return Ie.filter(t=>e?.[t]!==void 0).map(t=>`${t} ${e[t].ok?"ok":"NO"}`).join(" \xB7 ")}function $(e){return e!==null?String(Math.round(e*1e3)/1e3):"-"}function Pt(e){let t=W(JSON.stringify(e.data??{}),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${t}`.trimEnd()}function Pe(e){let t=e.failures?.length?` (${e.failures.length} task failure${e.failures.length===1?"":"s"})`:"";return `${e.message}${t}`}function ne(e){let t=[["id",e.id],["status",e.status]];if(e.benchmarkName!==void 0&&t.push(["benchmark",e.benchmarkName]),e.version!==void 0&&t.push(["version",e.version]),e.taskCount!==void 0&&t.push(["tasks",String(e.taskCount)]),e.failure){t.push(["failure",Pe(e.failure)]);for(let r of e.failure.failures??[])t.push([` ${r.taskKey}`,r.error]);}return h(t)}function Et(e){let t=[];return e.taskCount!==void 0&&t.push(`tasks=${e.taskCount}`),e.failure&&t.push(W(Pe(e.failure),140)),`status ${e.status.padEnd(12)} ${t.join(" ")}`.trimEnd()}function At(e){let t={...e.data??{}},r=[];typeof t.trialId=="string"&&r.push(t.trialId),typeof t.taskKey=="string"&&r.push(t.taskKey),typeof t.status=="string"&&r.push(t.status),typeof t.reward=="number"&&r.push(`reward=${t.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[i,o]of Object.entries(t))n.has(i)||r.push(`${i}=${typeof o=="object"?JSON.stringify(o):String(o)}`);let s=W(r.join(" "),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${s}`.trimEnd()}function b(e){let t={};return typeof e.flags["api-key"]=="string"&&(t.apiKey=e.flags["api-key"]),typeof e.flags["base-url"]=="string"&&(t.baseUrl=e.flags["base-url"]),t}function G(e){return {...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{}}}function Ot(e){return e.status==="COMPLETED"?0:e.status==="FAILED"||e.status==="CANCELLED"?1:0}async function Jt(e,t){let r=ht(e),n=e.flags.json===true,s=e.flags.watch===true,i=k(b(e)),o=await i.run(r);if(!s){if(n)t.out(JSON.stringify(o));else {for(let a of D(o))t.out(a);t.out(""),t.out(`Follow it with: evolve-evals get ${o.id}`);}return 0}n?t.out(JSON.stringify({kind:"job.created",job:o})):t.out(`Job ${o.id} (${o.benchmark}) ${o.status} \u2014 watching\u2026`);let u=await i.watch(o.id,{onEvent:a=>{t.out(n?JSON.stringify({kind:"event",...a}):At(a));}});if(n)t.out(JSON.stringify({kind:"job.final",job:u}));else {t.out("");for(let a of D(u))t.out(a);}return Ot(u)}async function _t(e,t){let n=await k(b(e)).list({...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(n)),0;if(n.items.length===0)return t.out("No jobs."),0;let s=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let i of n.items)s.push(vt(i));for(let i of h(s))t.out(i);return n.nextCursor&&t.out(`
|
|
88
|
+
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function Ut(e,t){let n=await k(b(e)).get(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Lt(e,t){let r=k(b(e)),n;if(e.flags.status!==void 0&&(n=String(e.flags.status).split(",").map(o=>o.trim()).filter(Boolean),n.length===0))throw new m("--status got an empty status list");let s=await r.trials(e.positionals[0],{...n!==void 0?{status:n}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(s)),0;if(s.items.length===0)return t.out("No trials."),0;let i=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let o of s.items)i.push(St(o));for(let o of h(i))t.out(o);return t.out(`
|
|
89
|
+
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function Dt(e,t){let n=await k(b(e)).trial(e.positionals[0],e.positionals[1]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Tt(n))t.out(s);return 0}async function $t(e,t){let r=k(b(e)),n=e.flags.json===true,s=0;for await(let i of r.trialTraceEvents(e.positionals[0],e.positionals[1],{...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{}}))t.out(n?JSON.stringify(i):Pt(i)),s+=1;return !n&&s===0&&t.out("No trace events."),0}function Bt(e){let t=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of e.jobs)r.push([n.id,n.benchmark,n.status,$(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,x(n.spentUsd)]);if(t.push(...h(r)),e.taskMatrix.length>0){t.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...e.jobs.map((i,o)=>`JOB ${o+1}`)]],s=e.jobs.map(i=>i.id);for(let i of e.taskMatrix){let o=new Map(i.cells.map(u=>[u.jobId,u]));n.push([i.taskKey,i.disagreement?"!":"",...s.map(u=>{let a=o.get(u);return a?a.meanReward!==null?`${a.status} ${$(a.meanReward)}`:a.status:"-"})]);}t.push(...h(n));}return t}async function Nt(e,t){let n=await k(b(e)).compare(e.positionals);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Bt(n))t.out(s);return 0}async function jt(e,t){let n=await k(b(e)).cancel(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Ht(e,t){let n=await k(b(e)).rerunFailed(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else {for(let s of D(n))t.out(s);t.out(""),t.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function Mt(e,t){let r=k(b(e)),[n,s]=e.positionals,i;if(s!==void 0){if(e.flags.status!==void 0||e.flags.task!==void 0)throw new m("--status/--task apply to a whole-job regrade, not a single trial");i=await r.regradeTrial(n,s);}else {let o={};if(e.flags.status!==void 0){let u=String(e.flags.status).split(",").map(a=>a.trim()).filter(Boolean);if(u.length===0)throw new m("--status got an empty status list");o.status=u;}e.flags.task!==void 0&&(o.taskKey=String(e.flags.task)),i=await r.regrade(n,o);}if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of xe(i))t.out(o);t.out(""),t.out(`Follow it with: evolve-evals regrade-job ${i.id}`);}return 0}async function Vt(e,t){let n=await k(b(e)).getRegrade(e.positionals[0],G(e));if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of xe(n))t.out(s);return 0}async function Kt(e,t){let r=e.flags.format;if(r!==void 0&&r!=="harbor")throw new m(`Unknown --format "${r}" (supported: harbor)`);let s=await k(b(e)).export(e.positionals[0],{to:e.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return e.flags.json===true?t.out(JSON.stringify({path:s})):t.out(`Saved ${s}`),0}async function Ft(e,t){let n=await q(b(e)).downloadPackage(e.positionals[0],{to:e.flags.to??process.cwd()});return e.flags.json===true?t.out(JSON.stringify({path:n})):t.out(`Saved ${n}`),0}function qt(e){let t=h([["name",e.name],["title",e.title??"-"],["description",e.description??"-"],["active version",e.activeVersion?.version??"-"]]);if(e.versions&&e.versions.length>0){t.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let n of e.versions)r.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);t.push(...h(r));}if(e.tasks&&e.tasks.items.length>0){t.push("",`Tasks (version ${e.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let s of e.tasks.items)r.push([s.taskKey,`${s.agentTimeoutSec}s`,`${s.verifierTimeoutSec}s`,It(s.providers)]);t.push(...h(r)),e.tasks.nextCursor&&t.push(`More tasks: evolve-evals benchmarks get ${e.name} --cursor ${e.tasks.nextCursor}`);let n=new Map;for(let s of e.tasks.items)for(let i of Ie){let o=s.providers?.[i];o&&!o.ok&&!n.has(`${i}:${o.reason}`)&&n.set(`${i}:${o.reason}`,`${i}: ${o.reason}`);}if(n.size>0){t.push("","Provider limitations:");for(let s of n.values())t.push(` ${s}`);}}return t}async function Gt(e,t){let r=q(b(e)),[n,s]=e.positionals;if(n===void 0){let o=await r.list(G(e));if(e.flags.json===true)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No benchmarks."),0;let u=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let a of o.items)u.push([a.name,a.activeVersion?.version??"-",a.activeVersion?.state??"-",a.activeVersion?String(a.activeVersion.taskCount):"-",a.title??"-"]);for(let a of h(u))t.out(a);for(let a of Ce(o.items))t.out(a);return 0}if(n!=="get")throw new m(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!s)throw new m('"benchmarks get" requires a <name[@version]> ref');let i=await r.get(s,G(e));if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of qt(i))t.out(o);for(let o of Ce([i]))t.out(o);}return 0}function Ce(e){let t=[];for(let r of e){if(!r.upstream?.moved)continue;let n=r.activeVersion?`@${r.activeVersion.version}`:"";t.push(`${r.name}${n} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return t}async function Wt(e,t){let[r,n]=e.positionals,s=e.flags.json===true,i=q(b(e));if(r==="status"){if(!n)throw new m('"import status" requires an <id>');let l=await i.getImport(n);if(s)t.out(JSON.stringify(l));else for(let d of ne(l))t.out(d);return 0}if(r!==void 0)throw new m(`Unknown import subcommand "${r}" (supported: status)`);let o=yt(e),u=await i.import(o);if(e.flags.watch!==true){if(s)t.out(JSON.stringify(u));else {for(let l of ne(u))t.out(l);t.out(""),t.out(`Follow it with: evolve-evals import status ${u.id}`);}return 0}s?t.out(JSON.stringify({kind:"import.created",benchmarkImport:u})):t.out(`Import ${u.id} (${o.benchmarkName}) ${u.status} \u2014 watching\u2026`);let a=await i.watchImport(u.id,{onStatus:l=>{t.out(s?JSON.stringify({kind:"import.status",benchmarkImport:l}):Et(l));}});if(s)t.out(JSON.stringify({kind:"import.final",benchmarkImport:a}));else {t.out("");for(let l of ne(a))t.out(l);}return a.status==="FAILED"?1:0}async function Yt(e,t){let r=ve(b(e)),[n,s]=e.positionals,i=e.flags.json===true;if(n===void 0){let o=await r.list(G(e));if(i)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No custom harnesses."),0;let u=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let a of o.items)u.push([a.name,a.source,W(a.runCommand,60),a.updatedAt]);for(let a of h(u))t.out(a);return 0}if(n==="add"){if(s!==void 0)throw new m(`"custom-harnesses add" got unexpected argument "${s}"`);let o=await r.create(Rt(e));if(i)t.out(JSON.stringify(o));else {for(let u of Te(o))t.out(u);t.out(""),t.out(`Use it with: evolve-evals run --agent ${o.name}:<model> \u2026`);}return 0}if(n==="get"){if(!s)throw new m('"custom-harnesses get" requires a <name>');let o=await r.get(s);if(i)t.out(JSON.stringify(o));else for(let u of Te(o))t.out(u);return 0}if(n==="remove"){if(!s)throw new m('"custom-harnesses remove" requires a <name>');return await r.delete(s),i?t.out(JSON.stringify({name:s,deleted:true})):t.out(`Deleted custom harness ${s}`),0}throw new m(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function zt(e,t=wt){let r;try{r=ft(e);}catch(n){if(n instanceof m)return t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(r.command){case "help":return t.out(mt),0;case "run":return await Jt(r,t);case "list":return await _t(r,t);case "get":return await Ut(r,t);case "trials":return await Lt(r,t);case "trial":return await Dt(r,t);case "trace":return await $t(r,t);case "compare":return await Nt(r,t);case "cancel":return await jt(r,t);case "rerun-failed":return await Ht(r,t);case "regrade":return await Mt(r,t);case "regrade-job":return await Vt(r,t);case "export":return await Kt(r,t);case "benchmarks":return await Gt(r,t);case "import":return await Wt(r,t);case "download":return await Ft(r,t);case "custom-harnesses":return await Yt(r,t);default:return t.err(`Error: unknown command "${r.command}"`),2}}catch(n){return n instanceof m?(t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2):(t.err(`Error: ${n.message}`),1)}}var Qt=(()=>{let e=process.argv[1];if(!e)return false;let t=r=>{try{return (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('cli.cjs', document.baseURI).href))===url.pathToFileURL(r).href}catch{return false}};if(t(e))return true;try{return t(fs.realpathSync(e))}catch{return false}})();Qt&&zt(process.argv.slice(2)).then(e=>{process.exitCode=e;},e=>{process.stderr.write(`Error: ${e?.message??e}
|
|
90
|
+
`),process.exitCode=1;});exports.CliUsageError=m;exports.USAGE=mt;exports.buildCustomHarnessInput=Rt;exports.buildImportInput=yt;exports.buildJobInput=ht;exports.eventLine=At;exports.importStatusLine=Et;exports.parseArgs=ft;exports.parseHarnessEnv=kt;exports.parseJobAgent=bt;exports.runCli=zt;exports.traceEventLine=Pt;
|
package/dist/hosted/cli.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {I as I$1,H as H$1,J as J$1}from'../chunk-ZYTUS4TY.js';import {readFileSync}from'fs';import {pathToFileURL}from'url';var
|
|
2
|
+
import {I as I$1,H as H$1,J as J$1}from'../chunk-ZYTUS4TY.js';import {realpathSync,readFileSync}from'fs';import {pathToFileURL}from'url';var J=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
3
3
|
|
|
4
4
|
Usage: evolve-evals <command> [options]
|
|
5
5
|
|
|
@@ -77,9 +77,9 @@ Other options:
|
|
|
77
77
|
--format harbor Export the Harbor job-layout bundle
|
|
78
78
|
--json Machine-readable JSON output
|
|
79
79
|
--api-key <key> API key (default: $EVOLVE_API_KEY)
|
|
80
|
-
--base-url <url> API base URL (default: the Evolve dashboard API)`,l=class extends Error{constructor(e){super(e),this.name="CliUsageError";}},
|
|
80
|
+
--base-url <url> API base URL (default: the Evolve dashboard API)`,l=class extends Error{constructor(e){super(e),this.name="CliUsageError";}},U={json:"boolean","api-key":"string","base-url":"string"},D={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},O={command:"help",positionals:[],flags:{}};function L(t){if(t.length===0)throw new l("No command given");if(t[0]==="--help"||t[0]==="-h")return O;let e=t[0],s=D[e];if(!s)throw new l(`Unknown command "${e}"`);let n={},r=[];for(let o=1;o<t.length;o++){let i=t[o];if(i==="--help"||i==="-h")return O;if(!i.startsWith("--")){r.push(i);continue}let a=i.slice(2),u,d=a.indexOf("=");d!==-1&&(u=a.slice(d+1),a=a.slice(0,d));let w=s.flags[a]??U[a];if(!w)throw new l(`Unknown option --${a} for "${e}"`);if(w==="boolean"){if(u!==void 0)throw new l(`Option --${a} takes no value`);n[a]=true;continue}let p=u;if(p===void 0){let g=t[o+1];if(g===void 0||g.startsWith("--"))throw new l(`Option --${a} requires a value`);p=g,o++;}if(w==="number"){let g=Number(p);if(p.trim()===""||!Number.isFinite(g))throw new l(`Option --${a} expects a number, got "${p}"`);n[a]=g;}else if(w==="repeat"){let g=n[a]??[];g.push(p),n[a]=g;}else n[a]=p;}if(r.length<s.minPositionals)throw new l(`"${e}" requires ${s.positionalUsage??`${s.minPositionals} argument(s)`}`);if(r.length>s.maxPositionals)throw new l(`"${e}" got unexpected argument "${r[s.maxPositionals]}"`);for(let o of s.required??[])if(!(o in n))throw new l(`"${e}" requires --${o}`);return {command:e,positionals:r,flags:n}}function M(t){let e=t.indexOf(":");if(e<=0||e===t.length-1)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);let s=t.slice(0,e),n=t.slice(e+1),r=n.indexOf(":");if(r===-1)return {harness:s,model:n};let o=n.slice(0,r),i=n.slice(r+1);if(!o||!i)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);return {harness:s,model:o,harnessVersion:i}}function q(t){let e=t.flags,s;if(e.tasks!==void 0&&(s=String(e.tasks).split(",").map(n=>n.trim()).filter(Boolean),s.length===0))throw new l("--tasks got an empty task list");return {benchmark:e.benchmark,...s!==void 0?{tasks:s}:{},agents:e.agent.map(n=>{let r=M(n);return e.effort!==void 0?{...r,reasoningEffort:e.effort}:r}),...e.runs!==void 0?{runsPerTask:e.runs}:{},...e.concurrency!==void 0?{concurrency:e.concurrency}:{},...e["max-trial-spend"]!==void 0?{maxTrialSpendUsd:e["max-trial-spend"]}:{},...e.provider!==void 0?{sandboxProvider:e.provider}:{}}}function K(t){let e=t.flags,s=typeof e.dir=="string",n=typeof e.git=="string"||typeof e.ref=="string";if(s&&n)throw new l('"import" takes EITHER --dir OR --git/--ref, not both');if(s){for(let r of ["name","version"])if(typeof e[r]!="string")throw new l(`"import" requires --${r}`);return {source:{directory:e.dir},benchmarkName:e.name,version:e.version}}for(let r of ["git","ref","name","version"])if(typeof e[r]!="string"){let o=r==="git"||r==="ref"?" (or --dir for a local corpus directory)":"";throw new l(`"import" requires --${r}${o}`)}return {source:{gitUrl:e.git,ref:e.ref},benchmarkName:e.name,version:e.version}}function V(t){let e={};for(let s of t){let n=s.indexOf("=");if(n<=0)throw new l(`Invalid --env "${s}": expected KEY=VALUE`);e[s.slice(0,n)]=s.slice(n+1);}return e}function F(t,e=s=>readFileSync(s,"utf-8")){let s=t.flags,n=typeof s.dir=="string",r=typeof s["install-script"]=="string";if(n&&r)throw new l('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!r)throw new l('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let i of ["name","run"])if(typeof s[i]!="string")throw new l(`"custom-harnesses add" requires --${i}`);let o=V(s.env??[]);return {name:s.name,...n?{directory:s.dir}:{installScript:e(s["install-script"])},runCommand:s.run,...Object.keys(o).length>0?{env:o}:{}}}var H={out:t=>process.stdout.write(t+`
|
|
81
81
|
`),err:t=>process.stderr.write(t+`
|
|
82
|
-
`)};function f(t){if(t.length===0)return [];let e=[];for(let
|
|
83
|
-
More: evolve-evals list --cursor ${
|
|
84
|
-
${r.items.length} trial(s) shown`),r.nextCursor&&e.out(`More: evolve-evals trials ${t.positionals[0]} --cursor ${r.nextCursor}`),0}async function
|
|
85
|
-
`),process.exitCode=1;});export{l as CliUsageError,
|
|
82
|
+
`)};function f(t){if(t.length===0)return [];let e=[];for(let s of t)s.forEach((n,r)=>{e[r]=Math.max(e[r]??0,n.length);});return t.map(s=>s.map((n,r)=>n.padEnd(e[r])).join(" ").trimEnd())}function h(t){return typeof t=="number"?`$${t.toFixed(2)}`:"-"}function $(t){let e=`${t.harness}:${t.model}`;return t.harnessVersion?`${e}:${t.harnessVersion}`:e}function S(t,e){return t.length>e?t.slice(0,e-1)+"\u2026":t}function b(t){let e=[["id",t.id],["status",t.status],["benchmark",t.benchmark]];e.push(["agents",t.agents.map($).join(", ")]),e.push(["size",`${t.counts.agents} agent(s) x ${t.counts.tasks} task(s) = ${t.trials.total} trial(s)`]),e.push(["runs/task",String(t.runsPerTask)]),e.push(["concurrency",String(t.concurrency)]),e.push(["max spend/trial",h(t.maxTrialSpendUsd)]),e.push(["worst case",h(t.worstCaseSpendUsd)]),e.push(["provider",t.sandboxProvider]),e.push(["spent",h(t.spentUsd)]),e.push(["mean reward",t.meanReward!==null?String(t.meanReward):"-"]);let s=Object.entries(t.trials.byStatus).filter(([,n])=>n>0).map(([n,r])=>`${n} ${r}`).join(" \xB7 ");return s&&e.push(["trials",s]),t.sourceJobId&&e.push(["rerun of",t.sourceJobId]),t.idempotentReplay&&e.push(["note","idempotent replay of an existing job"]),t.failure&&e.push(["failure",`${t.failure.code}: ${t.failure.message}`]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function B(t){return [t.id,t.status,t.benchmark,String(t.trials.total),k(t.meanReward),h(t.spentUsd),t.createdAt]}function G(t){return [t.taskKey,$(t.agent),String(t.runNumber),t.status,t.reward!==null?String(t.reward):"-",h(t.spentUsd),t.id]}function _(t){let e=[["trial id",t.id],["job",t.jobId],["task",t.taskKey],["agent",$(t.agent)],["run",String(t.runNumber)],["status",t.status],["reward",t.reward!==null?String(t.reward):"-"]];return t.metrics&&Object.keys(t.metrics).length>0&&e.push(["metrics",Object.entries(t.metrics).map(([s,n])=>`${s}=${n}`).join(" \xB7 ")]),e.push(["spent",h(t.spentUsd)]),t.sandboxProvider&&e.push(["provider",t.sandboxProvider]),t.verifierMode&&e.push(["verifier",t.verifierMode]),t.resolvedHarnessVersion&&e.push(["harness version",t.resolvedHarnessVersion]),t.phaseTimingsMs&&Object.keys(t.phaseTimingsMs).length>0&&e.push(["timings",Object.entries(t.phaseTimingsMs).map(([s,n])=>`${s}=${n}ms`).join(" \xB7 ")]),t.failurePhase&&e.push(["failure phase",t.failurePhase]),t.failureDetail&&e.push(["failure detail",t.failureDetail]),t.sessionRef&&e.push(["session",t.sessionRef]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function W(t){if(t===null)return "-";let e=Math.round(t*1e3)/1e3;return e>0?`+${e}`:String(e)}function Y(t){return [t.taskKey,t.status,k(t.sourceReward),k(t.reward),W(t.rewardDelta),t.sourceTrialId]}function x(t){let e=[["job id",t.id],["status",t.status],["source job",t.sourceJobId],["provider",t.sandboxProvider],["results",String(t.results.total)]],s=Object.entries(t.results.byStatus).filter(([,r])=>r>0).map(([r,o])=>`${r} ${o}`).join(" \xB7 ");if(s&&e.push(["by status",s]),t.filter&&(t.filter.status?.length||t.filter.taskKey)){let r=[];t.filter.status?.length&&r.push(`status=${t.filter.status.join(",")}`),t.filter.taskKey&&r.push(`task=${t.filter.taskKey}`),e.push(["filter",r.join(" \xB7 ")]);}e.push(["created",t.createdAt]);let n=f(e);if(t.results.items.length>0){n.push("");let r=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let o of t.results.items)r.push(Y(o));n.push(...f(r)),t.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${t.id} --cursor ${t.results.nextCursor}`);}return n}function P(t){let e=[["name",t.name],["source",t.source],["run command",t.runCommand]],s=Object.keys(t.env??{});return s.length>0&&e.push(["env",s.sort().join(", ")]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}var C=["e2b","daytona","modal"];function z(t){return C.filter(e=>t?.[e]!==void 0).map(e=>`${e} ${t[e].ok?"ok":"NO"}`).join(" \xB7 ")}function k(t){return t!==null?String(Math.round(t*1e3)/1e3):"-"}function Q(t){let e=S(JSON.stringify(t.data??{}),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${e}`.trimEnd()}function T(t){let e=t.failures?.length?` (${t.failures.length} task failure${t.failures.length===1?"":"s"})`:"";return `${t.message}${e}`}function I(t){let e=[["id",t.id],["status",t.status]];if(t.benchmarkName!==void 0&&e.push(["benchmark",t.benchmarkName]),t.version!==void 0&&e.push(["version",t.version]),t.taskCount!==void 0&&e.push(["tasks",String(t.taskCount)]),t.failure){e.push(["failure",T(t.failure)]);for(let s of t.failure.failures??[])e.push([` ${s.taskKey}`,s.error]);}return f(e)}function X(t){let e=[];return t.taskCount!==void 0&&e.push(`tasks=${t.taskCount}`),t.failure&&e.push(S(T(t.failure),140)),`status ${t.status.padEnd(12)} ${e.join(" ")}`.trimEnd()}function Z(t){let e={...t.data??{}},s=[];typeof e.trialId=="string"&&s.push(e.trialId),typeof e.taskKey=="string"&&s.push(e.taskKey),typeof e.status=="string"&&s.push(e.status),typeof e.reward=="number"&&s.push(`reward=${e.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[o,i]of Object.entries(e))n.has(o)||s.push(`${o}=${typeof i=="object"?JSON.stringify(i):String(i)}`);let r=S(s.join(" "),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${r}`.trimEnd()}function c(t){let e={};return typeof t.flags["api-key"]=="string"&&(e.apiKey=t.flags["api-key"]),typeof t.flags["base-url"]=="string"&&(e.baseUrl=t.flags["base-url"]),e}function v(t){return {...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{}}}function tt(t){return t.status==="COMPLETED"?0:t.status==="FAILED"||t.status==="CANCELLED"?1:0}async function et(t,e){let s=q(t),n=t.flags.json===true,r=t.flags.watch===true,o=J$1(c(t)),i=await o.run(s);if(!r){if(n)e.out(JSON.stringify(i));else {for(let u of b(i))e.out(u);e.out(""),e.out(`Follow it with: evolve-evals get ${i.id}`);}return 0}n?e.out(JSON.stringify({kind:"job.created",job:i})):e.out(`Job ${i.id} (${i.benchmark}) ${i.status} \u2014 watching\u2026`);let a=await o.watch(i.id,{onEvent:u=>{e.out(n?JSON.stringify({kind:"event",...u}):Z(u));}});if(n)e.out(JSON.stringify({kind:"job.final",job:a}));else {e.out("");for(let u of b(a))e.out(u);}return tt(a)}async function st(t,e){let n=await J$1(c(t)).list({...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(n)),0;if(n.items.length===0)return e.out("No jobs."),0;let r=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let o of n.items)r.push(B(o));for(let o of f(r))e.out(o);return n.nextCursor&&e.out(`
|
|
83
|
+
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function nt(t,e){let n=await J$1(c(t)).get(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of b(n))e.out(r);return 0}async function rt(t,e){let s=J$1(c(t)),n;if(t.flags.status!==void 0&&(n=String(t.flags.status).split(",").map(i=>i.trim()).filter(Boolean),n.length===0))throw new l("--status got an empty status list");let r=await s.trials(t.positionals[0],{...n!==void 0?{status:n}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(r)),0;if(r.items.length===0)return e.out("No trials."),0;let o=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let i of r.items)o.push(G(i));for(let i of f(o))e.out(i);return e.out(`
|
|
84
|
+
${r.items.length} trial(s) shown`),r.nextCursor&&e.out(`More: evolve-evals trials ${t.positionals[0]} --cursor ${r.nextCursor}`),0}async function it(t,e){let n=await J$1(c(t)).trial(t.positionals[0],t.positionals[1]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of _(n))e.out(r);return 0}async function ot(t,e){let s=J$1(c(t)),n=t.flags.json===true,r=0;for await(let o of s.trialTraceEvents(t.positionals[0],t.positionals[1],{...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{}}))e.out(n?JSON.stringify(o):Q(o)),r+=1;return !n&&r===0&&e.out("No trace events."),0}function at(t){let e=[],s=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of t.jobs)s.push([n.id,n.benchmark,n.status,k(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,h(n.spentUsd)]);if(e.push(...f(s)),t.taskMatrix.length>0){e.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...t.jobs.map((o,i)=>`JOB ${i+1}`)]],r=t.jobs.map(o=>o.id);for(let o of t.taskMatrix){let i=new Map(o.cells.map(a=>[a.jobId,a]));n.push([o.taskKey,o.disagreement?"!":"",...r.map(a=>{let u=i.get(a);return u?u.meanReward!==null?`${u.status} ${k(u.meanReward)}`:u.status:"-"})]);}e.push(...f(n));}return e}async function ut(t,e){let n=await J$1(c(t)).compare(t.positionals);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of at(n))e.out(r);return 0}async function lt(t,e){let n=await J$1(c(t)).cancel(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of b(n))e.out(r);return 0}async function ct(t,e){let n=await J$1(c(t)).rerunFailed(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(n));else {for(let r of b(n))e.out(r);e.out(""),e.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function ft(t,e){let s=J$1(c(t)),[n,r]=t.positionals,o;if(r!==void 0){if(t.flags.status!==void 0||t.flags.task!==void 0)throw new l("--status/--task apply to a whole-job regrade, not a single trial");o=await s.regradeTrial(n,r);}else {let i={};if(t.flags.status!==void 0){let a=String(t.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(a.length===0)throw new l("--status got an empty status list");i.status=a;}t.flags.task!==void 0&&(i.taskKey=String(t.flags.task)),o=await s.regrade(n,i);}if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of x(o))e.out(i);e.out(""),e.out(`Follow it with: evolve-evals regrade-job ${o.id}`);}return 0}async function dt(t,e){let n=await J$1(c(t)).getRegrade(t.positionals[0],v(t));if(t.flags.json===true)e.out(JSON.stringify(n));else for(let r of x(n))e.out(r);return 0}async function mt(t,e){let s=t.flags.format;if(s!==void 0&&s!=="harbor")throw new l(`Unknown --format "${s}" (supported: harbor)`);let r=await J$1(c(t)).export(t.positionals[0],{to:t.flags.to??process.cwd(),...s==="harbor"?{format:"harbor"}:{}});return t.flags.json===true?e.out(JSON.stringify({path:r})):e.out(`Saved ${r}`),0}async function gt(t,e){let n=await H$1(c(t)).downloadPackage(t.positionals[0],{to:t.flags.to??process.cwd()});return t.flags.json===true?e.out(JSON.stringify({path:n})):e.out(`Saved ${n}`),0}function pt(t){let e=f([["name",t.name],["title",t.title??"-"],["description",t.description??"-"],["active version",t.activeVersion?.version??"-"]]);if(t.versions&&t.versions.length>0){e.push("");let s=[["VERSION","STATE","TASKS","CREATED"]];for(let n of t.versions)s.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);e.push(...f(s));}if(t.tasks&&t.tasks.items.length>0){e.push("",`Tasks (version ${t.selectedVersion?.version??"?"}):`);let s=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let r of t.tasks.items)s.push([r.taskKey,`${r.agentTimeoutSec}s`,`${r.verifierTimeoutSec}s`,z(r.providers)]);e.push(...f(s)),t.tasks.nextCursor&&e.push(`More tasks: evolve-evals benchmarks get ${t.name} --cursor ${t.tasks.nextCursor}`);let n=new Map;for(let r of t.tasks.items)for(let o of C){let i=r.providers?.[o];i&&!i.ok&&!n.has(`${o}:${i.reason}`)&&n.set(`${o}:${i.reason}`,`${o}: ${i.reason}`);}if(n.size>0){e.push("","Provider limitations:");for(let r of n.values())e.push(` ${r}`);}}return e}async function ht(t,e){let s=H$1(c(t)),[n,r]=t.positionals;if(n===void 0){let i=await s.list(v(t));if(t.flags.json===true)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No benchmarks."),0;let a=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let u of i.items)a.push([u.name,u.activeVersion?.version??"-",u.activeVersion?.state??"-",u.activeVersion?String(u.activeVersion.taskCount):"-",u.title??"-"]);for(let u of f(a))e.out(u);for(let u of R(i.items))e.out(u);return 0}if(n!=="get")throw new l(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!r)throw new l('"benchmarks get" requires a <name[@version]> ref');let o=await s.get(r,v(t));if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of pt(o))e.out(i);for(let i of R([o]))e.out(i);}return 0}function R(t){let e=[];for(let s of t){if(!s.upstream?.moved)continue;let n=s.activeVersion?`@${s.activeVersion.version}`:"";e.push(`${s.name}${n} \xB7 upstream ${s.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${s.name} --version <new-version> --git-url <url> --ref ${s.upstream.ref}`);}return e}async function wt(t,e){let[s,n]=t.positionals,r=t.flags.json===true,o=H$1(c(t));if(s==="status"){if(!n)throw new l('"import status" requires an <id>');let d=await o.getImport(n);if(r)e.out(JSON.stringify(d));else for(let w of I(d))e.out(w);return 0}if(s!==void 0)throw new l(`Unknown import subcommand "${s}" (supported: status)`);let i=K(t),a=await o.import(i);if(t.flags.watch!==true){if(r)e.out(JSON.stringify(a));else {for(let d of I(a))e.out(d);e.out(""),e.out(`Follow it with: evolve-evals import status ${a.id}`);}return 0}r?e.out(JSON.stringify({kind:"import.created",benchmarkImport:a})):e.out(`Import ${a.id} (${i.benchmarkName}) ${a.status} \u2014 watching\u2026`);let u=await o.watchImport(a.id,{onStatus:d=>{e.out(r?JSON.stringify({kind:"import.status",benchmarkImport:d}):X(d));}});if(r)e.out(JSON.stringify({kind:"import.final",benchmarkImport:u}));else {e.out("");for(let d of I(u))e.out(d);}return u.status==="FAILED"?1:0}async function bt(t,e){let s=I$1(c(t)),[n,r]=t.positionals,o=t.flags.json===true;if(n===void 0){let i=await s.list(v(t));if(o)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No custom harnesses."),0;let a=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let u of i.items)a.push([u.name,u.source,S(u.runCommand,60),u.updatedAt]);for(let u of f(a))e.out(u);return 0}if(n==="add"){if(r!==void 0)throw new l(`"custom-harnesses add" got unexpected argument "${r}"`);let i=await s.create(F(t));if(o)e.out(JSON.stringify(i));else {for(let a of P(i))e.out(a);e.out(""),e.out(`Use it with: evolve-evals run --agent ${i.name}:<model> \u2026`);}return 0}if(n==="get"){if(!r)throw new l('"custom-harnesses get" requires a <name>');let i=await s.get(r);if(o)e.out(JSON.stringify(i));else for(let a of P(i))e.out(a);return 0}if(n==="remove"){if(!r)throw new l('"custom-harnesses remove" requires a <name>');return await s.delete(r),o?e.out(JSON.stringify({name:r,deleted:true})):e.out(`Deleted custom harness ${r}`),0}throw new l(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function kt(t,e=H){let s;try{s=L(t);}catch(n){if(n instanceof l)return e.err(`Error: ${n.message}`),e.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(s.command){case "help":return e.out(J),0;case "run":return await et(s,e);case "list":return await st(s,e);case "get":return await nt(s,e);case "trials":return await rt(s,e);case "trial":return await it(s,e);case "trace":return await ot(s,e);case "compare":return await ut(s,e);case "cancel":return await lt(s,e);case "rerun-failed":return await ct(s,e);case "regrade":return await ft(s,e);case "regrade-job":return await dt(s,e);case "export":return await mt(s,e);case "benchmarks":return await ht(s,e);case "import":return await wt(s,e);case "download":return await gt(s,e);case "custom-harnesses":return await bt(s,e);default:return e.err(`Error: unknown command "${s.command}"`),2}}catch(n){return n instanceof l?(e.err(`Error: ${n.message}`),e.err('Run "evolve-evals help" for usage.'),2):(e.err(`Error: ${n.message}`),1)}}var yt=(()=>{let t=process.argv[1];if(!t)return false;let e=s=>{try{return import.meta.url===pathToFileURL(s).href}catch{return false}};if(e(t))return true;try{return e(realpathSync(t))}catch{return false}})();yt&&kt(process.argv.slice(2)).then(t=>{process.exitCode=t;},t=>{process.stderr.write(`Error: ${t?.message??t}
|
|
85
|
+
`),process.exitCode=1;});export{l as CliUsageError,J as USAGE,F as buildCustomHarnessInput,K as buildImportInput,q as buildJobInput,Z as eventLine,X as importStatusLine,L as parseArgs,V as parseHarnessEnv,M as parseJobAgent,kt as runCli,Q as traceEventLine};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evolvingmachines/sdk",
|
|
3
|
-
"version": "0.0.52-project-sable.20260729.
|
|
3
|
+
"version": "0.0.52-project-sable.20260729.96caaab",
|
|
4
4
|
"keywords": [
|
|
5
5
|
"ai",
|
|
6
6
|
"agents",
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"type-check": "tsc --noEmit && tsc -p tsconfig.types-test.json",
|
|
48
48
|
"test": "npm run test:unit && npm run test:integration",
|
|
49
49
|
"pretest:unit": "npm run build",
|
|
50
|
-
"test:unit": "tsx tests/unit/semaphore.test.ts && tsx tests/unit/swarm-concurrency.test.ts && tsx tests/unit/swarm-verify.test.ts && tsx tests/unit/swarm-retry-verify.test.ts && tsx tests/unit/prompt-construction.test.ts && tsx tests/unit/observability-metadata.test.ts && tsx tests/unit/skills-integration.test.ts && tsx tests/unit/auth-config.test.ts && tsx tests/unit/browser-config.test.ts && tsx tests/unit/integrations-config.test.ts && tsx tests/unit/managed-secrets.test.ts && tsx tests/unit/plugins-config.test.ts && tsx tests/unit/session-runtime.test.ts && tsx tests/unit/cost-api.test.ts && tsx tests/unit/storage-config.test.ts && tsx tests/unit/checkpoint-tar.test.ts && tsx tests/unit/checkpoint-errors.test.ts && tsx tests/unit/checkpoint-flows.test.ts && tsx tests/unit/checkpoint-dx.test.ts && tsx tests/unit/checkpoint-edge-cases.test.ts && tsx tests/unit/codex-parser-errors.test.ts && tsx tests/unit/kimi-parser.test.ts && tsx tests/unit/kimi-mcp.test.ts && tsx tests/unit/opencode-parser.test.ts && tsx tests/unit/droid-parser.test.ts && tsx tests/unit/droid-mcp.test.ts && tsx tests/unit/provider-parity.test.ts && tsx tests/unit/storage-client.test.ts && tsx tests/unit/sessions-client.test.ts && tsx tests/unit/hosted-client.test.ts && tsx tests/unit/hosted-ergonomics.test.ts && tsx tests/unit/hosted-cli.test.ts && tsx tests/unit/hosted-types.test.ts && tsx tests/unit/hosted-error-codes.test.ts && tsx tests/unit/sandbox-artifacts.test.ts && tsx tests/unit/upload-file-from-path.test.ts",
|
|
50
|
+
"test:unit": "tsx tests/unit/semaphore.test.ts && tsx tests/unit/swarm-concurrency.test.ts && tsx tests/unit/swarm-verify.test.ts && tsx tests/unit/swarm-retry-verify.test.ts && tsx tests/unit/prompt-construction.test.ts && tsx tests/unit/observability-metadata.test.ts && tsx tests/unit/skills-integration.test.ts && tsx tests/unit/auth-config.test.ts && tsx tests/unit/browser-config.test.ts && tsx tests/unit/integrations-config.test.ts && tsx tests/unit/managed-secrets.test.ts && tsx tests/unit/plugins-config.test.ts && tsx tests/unit/session-runtime.test.ts && tsx tests/unit/cost-api.test.ts && tsx tests/unit/storage-config.test.ts && tsx tests/unit/checkpoint-tar.test.ts && tsx tests/unit/checkpoint-errors.test.ts && tsx tests/unit/checkpoint-flows.test.ts && tsx tests/unit/checkpoint-dx.test.ts && tsx tests/unit/checkpoint-edge-cases.test.ts && tsx tests/unit/codex-parser-errors.test.ts && tsx tests/unit/kimi-parser.test.ts && tsx tests/unit/kimi-mcp.test.ts && tsx tests/unit/opencode-parser.test.ts && tsx tests/unit/droid-parser.test.ts && tsx tests/unit/droid-mcp.test.ts && tsx tests/unit/provider-parity.test.ts && tsx tests/unit/storage-client.test.ts && tsx tests/unit/sessions-client.test.ts && tsx tests/unit/hosted-client.test.ts && tsx tests/unit/hosted-ergonomics.test.ts && tsx tests/unit/hosted-cli.test.ts && tsx tests/unit/hosted-cli-bin.test.ts && tsx tests/unit/hosted-types.test.ts && tsx tests/unit/hosted-error-codes.test.ts && tsx tests/unit/sandbox-artifacts.test.ts && tsx tests/unit/upload-file-from-path.test.ts",
|
|
51
51
|
"test:unit:upload-from-path": "tsx tests/unit/upload-file-from-path.test.ts",
|
|
52
52
|
"test:unit:parity": "tsx tests/unit/provider-parity.test.ts",
|
|
53
53
|
"test:unit:semaphore": "tsx tests/unit/semaphore.test.ts",
|
|
@@ -103,13 +103,14 @@
|
|
|
103
103
|
"test:gemini": "tsx tests/integration/01-all-agents-parallel.ts gemini",
|
|
104
104
|
"test:qwen": "tsx tests/integration/01-all-agents-parallel.ts qwen",
|
|
105
105
|
"test:unit:hosted-cli": "tsx tests/unit/hosted-cli.test.ts",
|
|
106
|
+
"test:unit:hosted-cli-bin": "tsx tests/unit/hosted-cli-bin.test.ts",
|
|
106
107
|
"test:unit:sandbox-artifacts": "tsx tests/unit/sandbox-artifacts.test.ts"
|
|
107
108
|
},
|
|
108
109
|
"dependencies": {
|
|
109
110
|
"@agentclientprotocol/sdk": "^0.5.1",
|
|
110
|
-
"@evolvingmachines/e2b": "^0.0.52-project-sable.20260729.
|
|
111
|
-
"@evolvingmachines/daytona": "^0.0.52-project-sable.20260729.
|
|
112
|
-
"@evolvingmachines/modal": "^0.0.52-project-sable.20260729.
|
|
111
|
+
"@evolvingmachines/e2b": "^0.0.52-project-sable.20260729.96caaab",
|
|
112
|
+
"@evolvingmachines/daytona": "^0.0.52-project-sable.20260729.96caaab",
|
|
113
|
+
"@evolvingmachines/modal": "^0.0.52-project-sable.20260729.96caaab",
|
|
113
114
|
"ajv": "^8.17.1",
|
|
114
115
|
"p-map": "^7.0.2",
|
|
115
116
|
"zod": "^3.24.0",
|