@evolvingmachines/sdk 0.0.52-project-sable.20260729.cc76df1 → 0.0.52-project-sable.20260730.b88a70e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BYR2FRZM.js +6 -0
- package/dist/hosted/cli.cjs +12 -10
- package/dist/hosted/cli.d.cts +8 -3
- package/dist/hosted/cli.d.ts +8 -3
- package/dist/hosted/cli.js +9 -7
- package/dist/index.cjs +64 -64
- package/dist/index.d.cts +79 -5
- package/dist/index.d.ts +79 -5
- package/dist/index.js +46 -46
- package/dist/{types-Dsz--evy.d.cts → types-BBerSem-.d.cts} +19 -8
- package/dist/{types-Dsz--evy.d.ts → types-BBerSem-.d.ts} +19 -8
- package/harness-capabilities.json +345 -0
- package/package.json +12 -8
- package/dist/chunk-TB5ZZSGF.js +0 -6
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import {createWriteStream}from'fs';import {mkdir,rename,rm}from'fs/promises';import {join,basename}from'path';import {Readable}from'stream';import {pipeline}from'stream/promises';function de(e=""){let t=process.env.EVOLVE_GATEWAY_URL||"https://swarmkit-gateway-692833842999.us-central1.run.app";if(!e)return t;let r=e.startsWith("/")?e:`/${e}`;return `${t}${r}`}function me(e=""){let t=process.env.EVOLVE_DASHBOARD_URL||w;if(!e)return t;let r=e.startsWith("/")?e:`/${e}`;return `${t.replace(/\/$/,"")}${r}`}var He=["e2b","daytona","modal"];function Y(e){return me(`/api/managed/${e}`)}function Me(){return `${Y("daytona")}/toolbox`}function Ne(){return Y("e2b")}function je(e){return {"browser-use":{type:"http",url:de("/browser_use/mcp"),headers:{"x-litellm-api-key":`Bearer ${e}`}}}}var N="EVOLVE_API_KEY",Ve="E2B_API_KEY",$e="DAYTONA_API_KEY",Ke="MODAL_TOKEN_ID",Fe="MODAL_TOKEN_SECRET",Ge="claude",qe=36e5,We=4,Ye=100,z="/home/user";function ze(e,t){return t?t.replace(/\/+$/,"")||"/":e?e==="root"?"/root":`/home/${e}`:z}var w=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai",Xe="x-litellm-customer-id",Ze="x-litellm-tags",et="run:",tt=".evolve-sdk/observability/sessions",rt=100,nt=5e3,st=3,at=1e3;var Q=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","upstream_not_watchable","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","package_not_retained","package_corrupt","package_missing","internal_error"];function j(e){return typeof e=="string"&&Q.includes(e)}var A=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,r,n,i){super(n),this.name="EvolveApiError",this.status=t,this.code=r,i?.param!==void 0&&(this.param=i.param),i?.details!==void 0&&(this.details=i.details),i?.retryAfterSec!==void 0&&(this.retryAfterSec=i.retryAfterSec),i?.requestId!==void 0&&(this.requestId=i.requestId);}isKnownCode(){return j(this.code)}};async function K(e){let t=await e.text().catch(()=>""),r=e.headers?.get?.("x-request-id")??void 0,n=Number(e.headers?.get?.("retry-after")),i=Number.isFinite(n)?n:void 0;try{let d=JSON.parse(t);if(d?.error&&typeof d.error=="object"){let c=typeof d.error.code=="string"?d.error.code:"unknown_error",l=typeof d.error.message=="string"?d.error.message:e.statusText;throw new A(e.status,c,l,{param:typeof d.error.param=="string"?d.error.param:void 0,details:d.error.details&&typeof d.error.details=="object"?d.error.details:void 0,retryAfterSec:typeof d.error.retryAfterSec=="number"?d.error.retryAfterSec:i,requestId:typeof d.error.requestId=="string"?d.error.requestId:r})}}catch(d){if(d instanceof A)throw d}throw new A(e.status,"unknown_error",t||e.statusText,{retryAfterSec:i,requestId:r})}var V=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},fe=1e3,he=3e4,Re=new Set(["COMPLETED","CANCELLED","FAILED"]),ye=new Set(["job.completed","job.cancelled","job.failed"]),ke=2e3,ve=new Set(["COMPLETED","FAILED"]);function F(e,t){let r=t?.apiKey||process.env[N];if(!r)throw new Error(`${e}() requires an API key. Set ${N} or pass { apiKey } in config.`);let n=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||w).replace(/\/$/,"");return {apiKey:r,baseUrl:n}}async function g(e,t,r){let n=await fetch(`${e.baseUrl}${t}`,{...r,headers:{Authorization:`Bearer ${e.apiKey}`,...r?.headers}});return n.ok||await K(n),n}function Te(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let r=e.slice(0,t).trim(),n=e.slice(t+1).trim();if(!r||!n)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:r,version:n}}function G(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function _(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function Ce(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function E(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(G),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function T(e,t){let r=e??{};return {items:Array.isArray(r.items)?r.items.map(t):[],nextCursor:r.nextCursor??null,hasMore:r.hasMore===true}}function v(e,t){let r=new URLSearchParams;e?.limit!==void 0&&r.set("limit",String(e.limit)),e?.cursor&&r.set("cursor",e.cursor);for(let[i,d]of Object.entries(t??{}))d!==void 0&&r.set(i,d);let n=r.toString();return n?`?${n}`:""}function X(e){return {id:e.id,taskKey:e.taskKey,agent:G(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sandboxId:e.sandboxId??null,verifierSandboxId:e.verifierSandboxId??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function xe(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function O(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...T(t,xe),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function J(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function B(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function oe(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function Se(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:oe(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(G),createdAt:e.createdAt}}function Pe(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:oe(e.coverage)}}function Ie(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(Pe)}}function Ee(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function $(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function Z(e,t){return t?.aborted?true:e?.name==="AbortError"}function D(e,t){return new Promise((r,n)=>{let i=setTimeout(()=>{t?.removeEventListener("abort",d),r();},e),d=()=>{clearTimeout(i),n(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",d,{once:true});})}function Ae(e){let t="";return {push(r){t+=r.replace(/\r\n/g,`
|
|
2
|
+
`);let n;for(;(n=t.indexOf(`
|
|
3
|
+
|
|
4
|
+
`))!==-1;){let i=t.slice(0,n);t=t.slice(n+2);let d,c,l=[];for(let u of i.split(`
|
|
5
|
+
`)){if(!u||u.startsWith(":"))continue;let a=u.indexOf(":"),s=a===-1?u:u.slice(0,a),o=a===-1?"":u.slice(a+1);o.startsWith(" ")&&(o=o.slice(1)),s==="id"?d=o:s==="event"?c=o:s==="data"&&l.push(o);}(d!==void 0||c!==void 0||l.length>0)&&e({id:d,event:c,data:l.join(`
|
|
6
|
+
`)});}}}}function x(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(i,d){return n().then(i,d)},catch(i){return n().catch(i)},finally(i){return n().finally(i)},async*[Symbol.asyncIterator](){let i=t?.cursor;for(;;){let d=await e({limit:t?.limit,cursor:i});for(let c of d.items)yield c;if(!d.nextCursor)return;i=d.nextCursor;}}}}function L(e,t){let r=new FormData;for(let[n,i]of Object.entries(e))i!==void 0&&r.set(n,i);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function we(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,i){return r().then(n,i)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var ie="x-package-sha256",U=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},H=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function _e(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new H(n,t)}async function Oe(e,t){let r=e.headers.get(ie);if(!r)return;let{createHash:n}=await import('crypto'),i=n("sha256").update(t).digest("hex");if(i!==r)throw new U(r,i)}function ue(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let i=basename(n[1]);return i===""||i==="."||i===".."||/[/\\]/.test(i)||/[\u0000-\u001f]/.test(i)?t:i}function Je(e){let t=F("benchmarks",e);async function r(c){let l=await g(t,`/api/benchmarks/imports/${encodeURIComponent(c)}`);return B(await l.json())}function n(c){if(!c||typeof c!="object")return null;let l=c;return {ref:l.ref,currentCommit:l.currentCommit,latestCommit:l.latestCommit??null,moved:l.moved===true,behindBy:typeof l.behindBy=="number"?l.behindBy:null,checkedAt:l.checkedAt??null,error:l.error??null}}async function i(c,l){let u=Te(c),a=v(l,{version:u.version}),o=await(await g(t,`/api/benchmarks/${encodeURIComponent(u.name)}${a}`)).json();return {name:o.name,title:o.title??null,description:o.description??null,activeVersion:o.activeVersion?_(o.activeVersion):null,versions:(o.versions||[]).map(_),selectedVersion:o.selectedVersion?_(o.selectedVersion):null,tasks:T(o.tasks,Ce),upstream:n(o.upstream),createdAt:o.createdAt,updatedAt:o.updatedAt}}async function d(c){let l=await g(t,`/api/benchmarks${v(c)}`);return T(await l.json(),u=>({name:u.name,title:u.title??null,description:u.description??null,activeVersion:u.activeVersion?_(u.activeVersion):null,upstream:n(u.upstream)}))}return {list(c){return x(d,c)},get:i,async getActive(c,l){let u=await i(c,l);if(u.activeVersion===null)throw new V(c);return {name:u.name,title:u.title,description:u.description,activeVersion:u.activeVersion,version:u.activeVersion.version,tasks:u.tasks??{items:[],nextCursor:null,hasMore:false},versions:u.versions??[],createdAt:u.createdAt,updatedAt:u.updatedAt}},async import(c){let l=c.source;if(l?.directory){let{tarGzipDirectory:u}=await import('./tar-WPIXS3E6.js'),a=u(l.directory),s=await g(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:c.benchmarkName,version:c.version},{bytes:a,filename:"corpus.tar.gz"})});return B(await s.json())}if(l&&"gitUrl"in l&&l.gitUrl){let u=await g(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:c.benchmarkName,version:c.version,gitUrl:l.gitUrl,ref:l.ref})});return B(await u.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(c,l){let u=l?.pollIntervalMs??ke,a=null;for(;;){$(l?.signal);let s=await r(c);if(s.status!==a&&(a=s.status,l?.onStatus?.(s)),ve.has(s.status))return s;await D(u,l?.signal);}},downloadPackage:(async(c,l)=>{let u=await g(t,`/api/benchmarks/imports/${encodeURIComponent(c)}/package`);if(l?.stream){if(!u.body)throw new Error("Package response has no body");return u.body}if(l?.to){if(!u.body)throw new Error("Package response has no body");let s=l.to;await mkdir(s,{recursive:true});let o=join(s,ue(u,`import-${c}-corpus.tar.gz`)),{createHash:m,randomBytes:b}=await import('crypto'),f=`${o}.${b(8).toString("hex")}.part`,y=m("sha256"),k=0,S=Readable.fromWeb(u.body);S.on("data",p=>{y.update(p),k+=p.length;});try{await pipeline(S,createWriteStream(f));let p=u.headers.get("Content-Length");if(p!==null&&k!==Number(p))throw new H(Number(p),k);let h=u.headers.get(ie),P=y.digest("hex");if(h&&P!==h)throw new U(h,P);await rename(f,o);}catch(p){throw await rm(f,{force:true}).catch(()=>{}),p}return o}let a=Buffer.from(await u.arrayBuffer());return _e(u,a.length),await Oe(u,a),a}),listImports(c){return x(async l=>{let u=new URLSearchParams;l.limit!==void 0&&u.set("limit",String(l.limit)),l.cursor!==void 0&&u.set("cursor",l.cursor),c?.status!==void 0&&u.set("status",c.status),c?.benchmark!==void 0&&u.set("benchmark",c.benchmark);let a=u.toString()?`?${u}`:"",s=await g(t,`/api/benchmarks/imports${a}`);return T(await s.json(),B)},c)},async delete(c){await g(t,`/api/benchmarks/${encodeURIComponent(c)}`,{method:"DELETE"});}}}function Be(e){let t=F("customHarnesses",e);return {async create(r){let n=await ee("customHarnesses().create()",r),i=await g(t,"/api/custom-harnesses",{method:"POST",body:n});return J(await i.json())},list(r){return x(async n=>{let i=await g(t,`/api/custom-harnesses${v(n)}`);return T(await i.json(),J)},r)},async get(r){let n=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return J(await n.json())},async delete(r){await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let i=await ee("customHarnesses().upsert()",{...n,name:r}),d=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:i});return J(await d.json())}}}async function ee(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let i={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:d}=await import('./tar-WPIXS3E6.js'),c=d(t.directory);return L(i,{bytes:c,filename:"source.tar.gz"})}return L(i)}function De(e){let t=F("jobs",e);async function r(a){let s=await g(t,`/api/jobs/${encodeURIComponent(a)}`);return E(await s.json())}async function n(a){let s=await g(t,`/api/jobs${v(a)}`);return T(await s.json(),E)}async function i(a,s){let o=v(s,{status:s?.status?.length?s.status.join(","):void 0}),m=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials${o}`);return T(await m.json(),X)}async function d(a,s,o){let m=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(s)}/trace${v(o)}`);return T(await m.json(),Ee)}async function c(a,s,o){let b=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(s)}/trace?stream=${o}`)).json();return o==="agent-home"?b.files??null:b.log??null}async function l(a,s){let o=s?`?format=${encodeURIComponent(s)}`:"";return g(t,`/api/jobs/${encodeURIComponent(a)}/export${o}`)}async function*u(a,s){let{onEvent:o,signal:m}=s??{},b=s?.reconnectDelayMs??fe,f=s?.maxReconnectDelayMs??he,y=null,k=false,S=false,p=b;for(;!k;){$(m);let h;try{h=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(a)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...y!==null?{"Last-Event-ID":String(y)}:{}},signal:m});}catch(R){if(Z(R,m))throw R;await D(p,m),p=Math.min(p*2,f);continue}if(!h.ok){if(h.status===429||h.status>=500){await h.text().catch(()=>""),await D(p,m),p=Math.min(p*2,f);continue}await K(h);}if(!h.body)throw new Error("Event stream response has no body");let P=false,M=[],le=Ae(R=>{let C=Number(R.id),I={seq:Number.isInteger(C)?C:-1,type:R.event||"message",data:R.data?Le(R.data):{}};Number.isInteger(C)&&(y=C),P=true,ye.has(I.type)&&(k=true),M.push(I);}),q=h.body.getReader(),ce=new TextDecoder;try{for(;!k;){let{done:R,value:C}=await q.read();if(R)break;le.push(ce.decode(C,{stream:!0}));for(let I of M)o?.(I),yield I;M.length=0;}}catch(R){if(Z(R,m))throw R}finally{await q.cancel().catch(()=>{});}if(k)break;$(m);let W=await r(a);if(Re.has(W.status)){if(S)return W;S=true;continue}P&&(p=b),await D(p,m),p=Math.min(p*2,f);}return r(a)}return {async run(a,s){let o=await g(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...s?.idempotencyKey?{"Idempotency-Key":s.idempotencyKey}:{}},body:JSON.stringify(a)});return E(await o.json())},get:r,list(a){return x(n,a)},trials(a,s){return x(o=>i(a,{...o,status:s?.status}),s)},async trial(a,s){let m=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(s)}`)).json();return {...X(m),jobId:m.jobId}},trialTrace:d,trialArtifact:c,async*trialTraceEvents(a,s,o){let m=o?.cursor;for(;;){let b=await d(a,s,{cursor:m,limit:o?.limit});for(let f of b.items)yield f;if(!b.nextCursor)return;m=b.nextCursor;}},watch(a,s){return we(u(a,s))},async cancel(a){let s=await g(t,`/api/jobs/${encodeURIComponent(a)}/cancel`,{method:"POST"});return E(await s.json())},async compare(a){let s=a.map(encodeURIComponent).join(","),m=await(await g(t,`/api/jobs/compare?ids=${s}`)).json();return {jobs:(m.jobs||[]).map(Se),taskMatrix:(m.taskMatrix||[]).map(Ie)}},async rerunFailed(a,s){let o=await g(t,`/api/jobs/${encodeURIComponent(a)}/rerun-failed`,{method:"POST",headers:s?.idempotencyKey?{"Idempotency-Key":s.idempotencyKey}:void 0});return E(await o.json())},async regrade(a,s){let o={};s?.status?.length&&(o.status=s.status),s?.taskKey!==void 0&&(o.taskKey=s.taskKey);let m=await g(t,`/api/jobs/${encodeURIComponent(a)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(o)});return O(await m.json())},async regradeTrial(a,s){let o=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(s)}/regrade`,{method:"POST"});return O(await o.json())},listRegrades(a){return x(async s=>{let m=await(await g(t,`/api/regrades${v(s,{jobId:a?.jobId})}`)).json();return {items:m.items.map(O),nextCursor:m.nextCursor,hasMore:m.hasMore}},a)},async getRegrade(a,s){let o=await g(t,`/api/regrades/${encodeURIComponent(a)}${v(s)}`);return O(await o.json())},export:(async(a,s)=>{let o=await l(a,s?.format);if(s?.stream){if(!o.body)throw new Error("Export response has no body");return o.body}if(s?.to){if(!o.body)throw new Error("Export response has no body");let b=s.to;await mkdir(b,{recursive:true});let f=join(b,ue(o,`job-${a}-export.json.gz`)),y=Readable.fromWeb(o.body);return await pipeline(y,createWriteStream(f)),f}let m=await o.arrayBuffer();return Buffer.from(m)})}}function Le(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}function bt(e){let t,r,n;return {get benchmarks(){return t??=Je(e)},get customHarnesses(){return r??=Be(e)},get jobs(){return n??=De(e)},meta(){return Ue(e)}}}async function Ue(e){let t=(e?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||w).replace(/\/$/,""),r=await fetch(`${t}/api/meta`);return r.ok||await K(r),await r.json()}export{at as A,Q as B,j as C,A as D,V as E,U as F,H as G,Je as H,Be as I,De as J,bt as K,Ue as L,de as a,me as b,He as c,Y as d,Me as e,Ne as f,je as g,N as h,Ve as i,$e as j,Ke as k,Fe as l,Ge as m,qe as n,We as o,Ye as p,z as q,ze as r,w as s,Xe as t,Ze as u,et as v,tt as w,rt as x,nt as y,st as z};
|
package/dist/hosted/cli.cjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
'use strict';var zlib=require('zlib'),fs=require('fs'),path=require('path'),url=require('url'),promises=require('fs/promises'),stream=require('stream'),promises$1=require('stream/promises');var _documentCurrentScript=typeof document!=='undefined'?document.currentScript:null;var
|
|
2
|
+
'use strict';var zlib=require('zlib'),fs=require('fs'),path=require('path'),url=require('url'),promises=require('fs/promises'),stream=require('stream'),promises$1=require('stream/promises');var _documentCurrentScript=typeof document!=='undefined'?document.currentScript:null;var Je=Object.defineProperty;var _e=(e,t)=>()=>(e&&(t=e(e=0)),t);var Ue=(e,t)=>{for(var r in t)Je(e,r,{get:t[r],enumerable:true});};var X={};Ue(X,{tarGzipDirectory:()=>Me});function je(e){let t=[],r=n=>{let s=n===""?e:path.join(e,n),o=fs.readdirSync(s,{withFileTypes:true}).sort((a,i)=>a.name<i.name?-1:a.name>i.name?1:0);for(let a of o){if(a.name.startsWith("."))continue;let i=n===""?a.name:`${n}/${a.name}`,u=path.join(s,a.name),l=fs.lstatSync(u);l.isSymbolicLink()||(l.isDirectory()?r(i):l.isFile()&&t.push({rel:i,abs:u}));}};return r(""),t.sort((n,s)=>n.rel<s.rel?-1:n.rel>s.rel?1:0)}function He(e,t){let r=e,n="";if(Buffer.byteLength(r,"utf8")>100){let a=-1;for(let i=r.indexOf("/");i!==-1;i=r.indexOf("/",i+1)){let u=r.slice(0,i),l=r.slice(i+1);l.length>0&&Buffer.byteLength(l,"utf8")<=100&&Buffer.byteLength(u,"utf8")<=155&&(a=i);}if(a===-1)throw new Error(`path too long to store in a USTAR tar (max 255 bytes, split at "/"): ${e}`);n=r.slice(0,a),r=r.slice(a+1);}let s=Buffer.alloc(E);s.write(r,0,100,"utf8"),s.write("0000644\0",100,"ascii"),s.write("0000000\0",108,"ascii"),s.write("0000000\0",116,"ascii"),s.write(t.toString(8).padStart(11,"0")+"\0",124,"ascii"),s.write("00000000000\0",136,"ascii"),s.write(" ",148,"ascii"),s.write("0",156,"ascii"),s.write("ustar\0",257,"ascii"),s.write("00",263,"ascii"),n&&s.write(n,345,155,"utf8");let o=0;for(let a=0;a<E;a++)o+=s[a];return s.write(o.toString(8).padStart(6,"0")+"\0 ",148,"ascii"),s}function Me(e){let t=je(e),r=[];for(let{rel:n,abs:s}of t){let o=fs.readFileSync(s);r.push(He(n,o.length)),r.push(o);let a=(E-o.length%E)%E;a>0&&r.push(Buffer.alloc(a));}return r.push(Buffer.alloc(E*2)),zlib.gzipSync(Buffer.concat(r),{level:9})}var E,Z=_e(()=>{E=512;});var z="EVOLVE_API_KEY";var ue=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai";var le=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","upstream_not_watchable","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","package_not_retained","package_corrupt","package_missing","internal_error"];function Q(e){return typeof e=="string"&&le.includes(e)}var $=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,r,n,s){super(n),this.name="EvolveApiError",this.status=t,this.code=r,s?.param!==void 0&&(this.param=s.param),s?.details!==void 0&&(this.details=s.details),s?.retryAfterSec!==void 0&&(this.retryAfterSec=s.retryAfterSec),s?.requestId!==void 0&&(this.requestId=s.requestId);}isKnownCode(){return Q(this.code)}};async function ke(e){let t=await e.text().catch(()=>""),r=e.headers?.get?.("x-request-id")??void 0,n=Number(e.headers?.get?.("retry-after")),s=Number.isFinite(n)?n:void 0;try{let o=JSON.parse(t);if(o?.error&&typeof o.error=="object"){let a=typeof o.error.code=="string"?o.error.code:"unknown_error",i=typeof o.error.message=="string"?o.error.message:e.statusText;throw new $(e.status,a,i,{param:typeof o.error.param=="string"?o.error.param:void 0,details:o.error.details&&typeof o.error.details=="object"?o.error.details:void 0,retryAfterSec:typeof o.error.retryAfterSec=="number"?o.error.retryAfterSec:s,requestId:typeof o.error.requestId=="string"?o.error.requestId:r})}}catch(o){if(o instanceof $)throw o}throw new $(e.status,"unknown_error",t||e.statusText,{retryAfterSec:s,requestId:r})}var ee=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},qe=1e3,Ge=3e4,We=new Set(["COMPLETED","CANCELLED","FAILED"]),Ye=new Set(["job.completed","job.cancelled","job.failed"]),ze=2e3,Qe=new Set(["COMPLETED","FAILED"]);function re(e,t){let r=t?.apiKey||process.env[z];if(!r)throw new Error(`${e}() requires an API key. Set ${z} or pass { apiKey } in config.`);let n=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||ue).replace(/\/$/,"");return {apiKey:r,baseUrl:n}}async function p(e,t,r){let n=await fetch(`${e.baseUrl}${t}`,{...r,headers:{Authorization:`Bearer ${e.apiKey}`,...r?.headers}});return n.ok||await ke(n),n}function Xe(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let r=e.slice(0,t).trim(),n=e.slice(t+1).trim();if(!r||!n)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:r,version:n}}function ne(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function B(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function Ze(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function U(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(ne),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function T(e,t){let r=e??{};return {items:Array.isArray(r.items)?r.items.map(t):[],nextCursor:r.nextCursor??null,hasMore:r.hasMore===true}}function x(e,t){let r=new URLSearchParams;e?.limit!==void 0&&r.set("limit",String(e.limit)),e?.cursor&&r.set("cursor",e.cursor);for(let[s,o]of Object.entries(t??{}))o!==void 0&&r.set(s,o);let n=r.toString();return n?`?${n}`:""}function de(e){return {id:e.id,taskKey:e.taskKey,agent:ne(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sandboxId:e.sandboxId??null,verifierSandboxId:e.verifierSandboxId??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function et(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function N(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...T(t,et),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function j(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function H(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function Re(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function tt(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:Re(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(ne),createdAt:e.createdAt}}function rt(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:Re(e.coverage)}}function nt(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(rt)}}function st(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function te(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function me(e,t){return t?.aborted?true:e?.name==="AbortError"}function M(e,t){return new Promise((r,n)=>{let s=setTimeout(()=>{t?.removeEventListener("abort",o),r();},e),o=()=>{clearTimeout(s),n(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",o,{once:true});})}function at(e){let t="";return {push(r){t+=r.replace(/\r\n/g,`
|
|
3
3
|
`);let n;for(;(n=t.indexOf(`
|
|
4
4
|
|
|
5
|
-
`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let
|
|
6
|
-
`)){if(!
|
|
7
|
-
`)});}}}}function J(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,i){return n().then(s,i)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let i=await e({limit:t?.limit,cursor:s});for(let o of i.items)yield o;if(!i.nextCursor)return;s=i.nextCursor;}}}}function V(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function at(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var Re="x-package-sha256",K=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},F=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function ot(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new F(n,t)}async function it(e,t){let r=e.headers.get(Re);if(!r)return;let{createHash:n}=await import('crypto'),s=n("sha256").update(t).digest("hex");if(s!==r)throw new K(r,s)}function ve(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let s=path.basename(n[1]);return s===""||s==="."||s===".."||/[/\\]/.test(s)||/[\u0000-\u001f]/.test(s)?t:s}function q(e){let t=te("benchmarks",e);async function r(o){let u=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}`);return H(await u.json())}function n(o){if(!o||typeof o!="object")return null;let u=o;return {ref:u.ref,currentCommit:u.currentCommit,latestCommit:u.latestCommit??null,moved:u.moved===true,behindBy:typeof u.behindBy=="number"?u.behindBy:null,checkedAt:u.checkedAt??null,error:u.error??null}}async function s(o,u){let a=Qe(o),l=S(u,{version:a.version}),c=await(await g(t,`/api/benchmarks/${encodeURIComponent(a.name)}${l}`)).json();return {name:c.name,title:c.title??null,description:c.description??null,activeVersion:c.activeVersion?B(c.activeVersion):null,versions:(c.versions||[]).map(B),selectedVersion:c.selectedVersion?B(c.selectedVersion):null,tasks:x(c.tasks,Xe),upstream:n(c.upstream),createdAt:c.createdAt,updatedAt:c.updatedAt}}async function i(o){let u=await g(t,`/api/benchmarks${S(o)}`);return x(await u.json(),a=>({name:a.name,title:a.title??null,description:a.description??null,activeVersion:a.activeVersion?B(a.activeVersion):null,upstream:n(a.upstream)}))}return {list(o){return J(i,o)},get:s,async getActive(o,u){let a=await s(o,u);if(a.activeVersion===null)throw new Z(o);return {name:a.name,title:a.title,description:a.description,activeVersion:a.activeVersion,version:a.activeVersion.version,tasks:a.tasks??{items:[],nextCursor:null,hasMore:false},versions:a.versions??[],createdAt:a.createdAt,updatedAt:a.updatedAt}},async import(o){let u=o.source;if(u?.directory){let{tarGzipDirectory:a}=await Promise.resolve().then(()=>(X(),Q)),l=a(u.directory),d=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version},{bytes:l,filename:"corpus.tar.gz"})});return H(await d.json())}if(u&&"gitUrl"in u&&u.gitUrl){let a=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version,gitUrl:u.gitUrl,ref:u.ref})});return H(await a.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(o,u){let a=u?.pollIntervalMs??Ye,l=null;for(;;){ee(u?.signal);let d=await r(o);if(d.status!==l&&(l=d.status,u?.onStatus?.(d)),ze.has(d.status))return d;await M(a,u?.signal);}},downloadPackage:(async(o,u)=>{let a=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}/package`);if(u?.stream){if(!a.body)throw new Error("Package response has no body");return a.body}if(u?.to){if(!a.body)throw new Error("Package response has no body");let d=u.to;await promises.mkdir(d,{recursive:true});let c=path.join(d,ve(a,`import-${o}-corpus.tar.gz`)),{createHash:p,randomBytes:v}=await import('crypto'),w=`${c}.${v(8).toString("hex")}.part`,I=p("sha256"),C=0,y=stream.Readable.fromWeb(a.body);y.on("data",f=>{I.update(f),C+=f.length;});try{await promises$1.pipeline(y,fs.createWriteStream(w));let f=a.headers.get("Content-Length");if(f!==null&&C!==Number(f))throw new F(Number(f),C);let P=a.headers.get(Re),E=I.digest("hex");if(P&&E!==P)throw new K(P,E);await promises.rename(w,c);}catch(f){throw await promises.rm(w,{force:true}).catch(()=>{}),f}return c}let l=Buffer.from(await a.arrayBuffer());return ot(a,l.length),await it(a,l),l}),listImports(o){return J(async u=>{let a=new URLSearchParams;u.limit!==void 0&&a.set("limit",String(u.limit)),u.cursor!==void 0&&a.set("cursor",u.cursor),o?.status!==void 0&&a.set("status",o.status),o?.benchmark!==void 0&&a.set("benchmark",o.benchmark);let l=a.toString()?`?${a}`:"",d=await g(t,`/api/benchmarks/imports${l}`);return x(await d.json(),H)},o)},async delete(o){await g(t,`/api/benchmarks/${encodeURIComponent(o)}`,{method:"DELETE"});}}}function we(e){let t=te("customHarnesses",e);return {async create(r){let n=await me("customHarnesses().create()",r),s=await g(t,"/api/custom-harnesses",{method:"POST",body:n});return j(await s.json())},list(r){return J(async n=>{let s=await g(t,`/api/custom-harnesses${S(n)}`);return x(await s.json(),j)},r)},async get(r){let n=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return j(await n.json())},async delete(r){await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await me("customHarnesses().upsert()",{...n,name:r}),i=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return j(await i.json())}}}async function me(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:i}=await Promise.resolve().then(()=>(X(),Q)),o=i(t.directory);return V(s,{bytes:o,filename:"source.tar.gz"})}return V(s)}function k(e){let t=te("jobs",e);async function r(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}`);return U(await l.json())}async function n(a){let l=await g(t,`/api/jobs${S(a)}`);return x(await l.json(),U)}async function s(a,l){let d=S(l,{status:l?.status?.length?l.status.join(","):void 0}),c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials${d}`);return x(await c.json(),ce)}async function i(a,l,d){let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/trace${S(d)}`);return x(await c.json(),nt)}async function o(a,l){let d=l?`?format=${encodeURIComponent(l)}`:"";return g(t,`/api/jobs/${encodeURIComponent(a)}/export${d}`)}async function*u(a,l){let{onEvent:d,signal:c}=l??{},p=l?.reconnectDelayMs??Fe,v=l?.maxReconnectDelayMs??qe,w=null,I=false,C=false,y=p;for(;!I;){ee(c);let f;try{f=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(a)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...w!==null?{"Last-Event-ID":String(w)}:{}},signal:c});}catch(R){if(de(R,c))throw R;await M(y,c),y=Math.min(y*2,v);continue}if(!f.ok){if(f.status===429||f.status>=500){await f.text().catch(()=>""),await M(y,c),y=Math.min(y*2,v);continue}await ye(f);}if(!f.body)throw new Error("Event stream response has no body");let P=false,E=[],Ee=st(R=>{let A=Number(R.id),_={seq:Number.isInteger(A)?A:-1,type:R.event||"message",data:R.data?ut(R.data):{}};Number.isInteger(A)&&(w=A),P=true,We.has(_.type)&&(I=true),E.push(_);}),ae=f.body.getReader(),Ae=new TextDecoder;try{for(;!I;){let{done:R,value:A}=await ae.read();if(R)break;Ee.push(Ae.decode(A,{stream:!0}));for(let _ of E)d?.(_),yield _;E.length=0;}}catch(R){if(de(R,c))throw R}finally{await ae.cancel().catch(()=>{});}if(I)break;ee(c);let oe=await r(a);if(Ge.has(oe.status)){if(C)return oe;C=true;continue}P&&(y=p),await M(y,c),y=Math.min(y*2,v);}return r(a)}return {async run(a,l){let d=await g(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:{}},body:JSON.stringify(a)});return U(await d.json())},get:r,list(a){return J(n,a)},trials(a,l){return J(d=>s(a,{...d,status:l?.status}),l)},async trial(a,l){let c=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}`)).json();return {...ce(c),jobId:c.jobId}},trialTrace:i,async*trialTraceEvents(a,l,d){let c=d?.cursor;for(;;){let p=await i(a,l,{cursor:c,limit:d?.limit});for(let v of p.items)yield v;if(!p.nextCursor)return;c=p.nextCursor;}},watch(a,l){return at(u(a,l))},async cancel(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}/cancel`,{method:"POST"});return U(await l.json())},async compare(a){let l=a.map(encodeURIComponent).join(","),c=await(await g(t,`/api/jobs/compare?ids=${l}`)).json();return {jobs:(c.jobs||[]).map(et),taskMatrix:(c.taskMatrix||[]).map(rt)}},async rerunFailed(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/rerun-failed`,{method:"POST",headers:l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:void 0});return U(await d.json())},async regrade(a,l){let d={};l?.status?.length&&(d.status=l.status),l?.taskKey!==void 0&&(d.taskKey=l.taskKey);let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return N(await c.json())},async regradeTrial(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/regrade`,{method:"POST"});return N(await d.json())},listRegrades(a){return J(async l=>{let c=await(await g(t,`/api/regrades${S(l,{jobId:a?.jobId})}`)).json();return {items:c.items.map(N),nextCursor:c.nextCursor,hasMore:c.hasMore}},a)},async getRegrade(a,l){let d=await g(t,`/api/regrades/${encodeURIComponent(a)}${S(l)}`);return N(await d.json())},export:(async(a,l)=>{let d=await o(a,l?.format);if(l?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(l?.to){if(!d.body)throw new Error("Export response has no body");let p=l.to;await promises.mkdir(p,{recursive:true});let v=path.join(p,ve(d,`job-${a}-export.json.gz`)),w=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(w,fs.createWriteStream(v)),v}let c=await d.arrayBuffer();return Buffer.from(c)})}}function ut(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var mt=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
5
|
+
`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let o,a,i=[];for(let u of s.split(`
|
|
6
|
+
`)){if(!u||u.startsWith(":"))continue;let l=u.indexOf(":"),c=l===-1?u:u.slice(0,l),d=l===-1?"":u.slice(l+1);d.startsWith(" ")&&(d=d.slice(1)),c==="id"?o=d:c==="event"?a=d:c==="data"&&i.push(d);}(o!==void 0||a!==void 0||i.length>0)&&e({id:o,event:a,data:i.join(`
|
|
7
|
+
`)});}}}}function A(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,o){return n().then(s,o)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let o=await e({limit:t?.limit,cursor:s});for(let a of o.items)yield a;if(!o.nextCursor)return;s=o.nextCursor;}}}}function V(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function ot(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var we="x-package-sha256",K=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},F=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function it(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new F(n,t)}async function ut(e,t){let r=e.headers.get(we);if(!r)return;let{createHash:n}=await import('crypto'),s=n("sha256").update(t).digest("hex");if(s!==r)throw new K(r,s)}function ve(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let s=path.basename(n[1]);return s===""||s==="."||s===".."||/[/\\]/.test(s)||/[\u0000-\u001f]/.test(s)?t:s}function q(e){let t=re("benchmarks",e);async function r(a){let i=await p(t,`/api/benchmarks/imports/${encodeURIComponent(a)}`);return H(await i.json())}function n(a){if(!a||typeof a!="object")return null;let i=a;return {ref:i.ref,currentCommit:i.currentCommit,latestCommit:i.latestCommit??null,moved:i.moved===true,behindBy:typeof i.behindBy=="number"?i.behindBy:null,checkedAt:i.checkedAt??null,error:i.error??null}}async function s(a,i){let u=Xe(a),l=x(i,{version:u.version}),d=await(await p(t,`/api/benchmarks/${encodeURIComponent(u.name)}${l}`)).json();return {name:d.name,title:d.title??null,description:d.description??null,activeVersion:d.activeVersion?B(d.activeVersion):null,versions:(d.versions||[]).map(B),selectedVersion:d.selectedVersion?B(d.selectedVersion):null,tasks:T(d.tasks,Ze),upstream:n(d.upstream),createdAt:d.createdAt,updatedAt:d.updatedAt}}async function o(a){let i=await p(t,`/api/benchmarks${x(a)}`);return T(await i.json(),u=>({name:u.name,title:u.title??null,description:u.description??null,activeVersion:u.activeVersion?B(u.activeVersion):null,upstream:n(u.upstream)}))}return {list(a){return A(o,a)},get:s,async getActive(a,i){let u=await s(a,i);if(u.activeVersion===null)throw new ee(a);return {name:u.name,title:u.title,description:u.description,activeVersion:u.activeVersion,version:u.activeVersion.version,tasks:u.tasks??{items:[],nextCursor:null,hasMore:false},versions:u.versions??[],createdAt:u.createdAt,updatedAt:u.updatedAt}},async import(a){let i=a.source;if(i?.directory){let{tarGzipDirectory:u}=await Promise.resolve().then(()=>(Z(),X)),l=u(i.directory),c=await p(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:a.benchmarkName,version:a.version},{bytes:l,filename:"corpus.tar.gz"})});return H(await c.json())}if(i&&"gitUrl"in i&&i.gitUrl){let u=await p(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:a.benchmarkName,version:a.version,gitUrl:i.gitUrl,ref:i.ref})});return H(await u.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(a,i){let u=i?.pollIntervalMs??ze,l=null;for(;;){te(i?.signal);let c=await r(a);if(c.status!==l&&(l=c.status,i?.onStatus?.(c)),Qe.has(c.status))return c;await M(u,i?.signal);}},downloadPackage:(async(a,i)=>{let u=await p(t,`/api/benchmarks/imports/${encodeURIComponent(a)}/package`);if(i?.stream){if(!u.body)throw new Error("Package response has no body");return u.body}if(i?.to){if(!u.body)throw new Error("Package response has no body");let c=i.to;await promises.mkdir(c,{recursive:true});let d=path.join(c,ve(u,`import-${a}-corpus.tar.gz`)),{createHash:m,randomBytes:f}=await import('crypto'),h=`${d}.${f(8).toString("hex")}.part`,S=m("sha256"),I=0,O=stream.Readable.fromWeb(u.body);O.on("data",b=>{S.update(b),I+=b.length;});try{await promises$1.pipeline(O,fs.createWriteStream(h));let b=u.headers.get("Content-Length");if(b!==null&&I!==Number(b))throw new F(Number(b),I);let w=u.headers.get(we),J=S.digest("hex");if(w&&J!==w)throw new K(w,J);await promises.rename(h,d);}catch(b){throw await promises.rm(h,{force:true}).catch(()=>{}),b}return d}let l=Buffer.from(await u.arrayBuffer());return it(u,l.length),await ut(u,l),l}),listImports(a){return A(async i=>{let u=new URLSearchParams;i.limit!==void 0&&u.set("limit",String(i.limit)),i.cursor!==void 0&&u.set("cursor",i.cursor),a?.status!==void 0&&u.set("status",a.status),a?.benchmark!==void 0&&u.set("benchmark",a.benchmark);let l=u.toString()?`?${u}`:"",c=await p(t,`/api/benchmarks/imports${l}`);return T(await c.json(),H)},a)},async delete(a){await p(t,`/api/benchmarks/${encodeURIComponent(a)}`,{method:"DELETE"});}}}function Se(e){let t=re("customHarnesses",e);return {async create(r){let n=await ge("customHarnesses().create()",r),s=await p(t,"/api/custom-harnesses",{method:"POST",body:n});return j(await s.json())},list(r){return A(async n=>{let s=await p(t,`/api/custom-harnesses${x(n)}`);return T(await s.json(),j)},r)},async get(r){let n=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return j(await n.json())},async delete(r){await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await ge("customHarnesses().upsert()",{...n,name:r}),o=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return j(await o.json())}}}async function ge(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:o}=await Promise.resolve().then(()=>(Z(),X)),a=o(t.directory);return V(s,{bytes:a,filename:"source.tar.gz"})}return V(s)}function R(e){let t=re("jobs",e);async function r(l){let c=await p(t,`/api/jobs/${encodeURIComponent(l)}`);return U(await c.json())}async function n(l){let c=await p(t,`/api/jobs${x(l)}`);return T(await c.json(),U)}async function s(l,c){let d=x(c,{status:c?.status?.length?c.status.join(","):void 0}),m=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials${d}`);return T(await m.json(),de)}async function o(l,c,d){let m=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/trace${x(d)}`);return T(await m.json(),st)}async function a(l,c,d){let f=await(await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/trace?stream=${d}`)).json();return d==="agent-home"?f.files??null:f.log??null}async function i(l,c){let d=c?`?format=${encodeURIComponent(c)}`:"";return p(t,`/api/jobs/${encodeURIComponent(l)}/export${d}`)}async function*u(l,c){let{onEvent:d,signal:m}=c??{},f=c?.reconnectDelayMs??qe,h=c?.maxReconnectDelayMs??Ge,S=null,I=false,O=false,b=f;for(;!I;){te(m);let w;try{w=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(l)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...S!==null?{"Last-Event-ID":String(S)}:{}},signal:m});}catch(v){if(me(v,m))throw v;await M(b,m),b=Math.min(b*2,h);continue}if(!w.ok){if(w.status===429||w.status>=500){await w.text().catch(()=>""),await M(b,m),b=Math.min(b*2,h);continue}await ke(w);}if(!w.body)throw new Error("Event stream response has no body");let J=false,Y=[],Ae=at(v=>{let P=Number(v.id),_={seq:Number.isInteger(P)?P:-1,type:v.event||"message",data:v.data?lt(v.data):{}};Number.isInteger(P)&&(S=P),J=true,Ye.has(_.type)&&(I=true),Y.push(_);}),oe=w.body.getReader(),Oe=new TextDecoder;try{for(;!I;){let{done:v,value:P}=await oe.read();if(v)break;Ae.push(Oe.decode(P,{stream:!0}));for(let _ of Y)d?.(_),yield _;Y.length=0;}}catch(v){if(me(v,m))throw v}finally{await oe.cancel().catch(()=>{});}if(I)break;te(m);let ie=await r(l);if(We.has(ie.status)){if(O)return ie;O=true;continue}J&&(b=f),await M(b,m),b=Math.min(b*2,h);}return r(l)}return {async run(l,c){let d=await p(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...c?.idempotencyKey?{"Idempotency-Key":c.idempotencyKey}:{}},body:JSON.stringify(l)});return U(await d.json())},get:r,list(l){return A(n,l)},trials(l,c){return A(d=>s(l,{...d,status:c?.status}),c)},async trial(l,c){let m=await(await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}`)).json();return {...de(m),jobId:m.jobId}},trialTrace:o,trialArtifact:a,async*trialTraceEvents(l,c,d){let m=d?.cursor;for(;;){let f=await o(l,c,{cursor:m,limit:d?.limit});for(let h of f.items)yield h;if(!f.nextCursor)return;m=f.nextCursor;}},watch(l,c){return ot(u(l,c))},async cancel(l){let c=await p(t,`/api/jobs/${encodeURIComponent(l)}/cancel`,{method:"POST"});return U(await c.json())},async compare(l){let c=l.map(encodeURIComponent).join(","),m=await(await p(t,`/api/jobs/compare?ids=${c}`)).json();return {jobs:(m.jobs||[]).map(tt),taskMatrix:(m.taskMatrix||[]).map(nt)}},async rerunFailed(l,c){let d=await p(t,`/api/jobs/${encodeURIComponent(l)}/rerun-failed`,{method:"POST",headers:c?.idempotencyKey?{"Idempotency-Key":c.idempotencyKey}:void 0});return U(await d.json())},async regrade(l,c){let d={};c?.status?.length&&(d.status=c.status),c?.taskKey!==void 0&&(d.taskKey=c.taskKey);let m=await p(t,`/api/jobs/${encodeURIComponent(l)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return N(await m.json())},async regradeTrial(l,c){let d=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/regrade`,{method:"POST"});return N(await d.json())},listRegrades(l){return A(async c=>{let m=await(await p(t,`/api/regrades${x(c,{jobId:l?.jobId})}`)).json();return {items:m.items.map(N),nextCursor:m.nextCursor,hasMore:m.hasMore}},l)},async getRegrade(l,c){let d=await p(t,`/api/regrades/${encodeURIComponent(l)}${x(c)}`);return N(await d.json())},export:(async(l,c)=>{let d=await i(l,c?.format);if(c?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(c?.to){if(!d.body)throw new Error("Export response has no body");let f=c.to;await promises.mkdir(f,{recursive:true});let h=path.join(f,ve(d,`job-${l}-export.json.gz`)),S=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(S,fs.createWriteStream(h)),h}let m=await d.arrayBuffer();return Buffer.from(m)})}}function lt(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var gt=`evolve-evals \u2014 Evolve hosted jobs CLI
|
|
8
8
|
|
|
9
9
|
Usage: evolve-evals <command> [options]
|
|
10
10
|
|
|
@@ -14,7 +14,7 @@ Commands:
|
|
|
14
14
|
get <id> Show one job
|
|
15
15
|
trials <id> List a job's trials
|
|
16
16
|
trial <id> <trial-id> Show one trial in full detail
|
|
17
|
-
trace <id> <trial-id> Print a trial's trace events
|
|
17
|
+
trace <id> <trial-id> Print a trial's trace events (--stream <raw-artifact>, --save <dir> for all)
|
|
18
18
|
compare <id> <id> [...] Compare 2-5 jobs side by side
|
|
19
19
|
cancel <id> Request cancellation of a job
|
|
20
20
|
rerun-failed <id> New job from a terminal job's failed trials
|
|
@@ -82,9 +82,11 @@ Other options:
|
|
|
82
82
|
--format harbor Export the Harbor job-layout bundle
|
|
83
83
|
--json Machine-readable JSON output
|
|
84
84
|
--api-key <key> API key (default: $EVOLVE_API_KEY)
|
|
85
|
-
--base-url <url> API base URL (default: the Evolve dashboard API)`,
|
|
85
|
+
--base-url <url> API base URL (default: the Evolve dashboard API)`,g=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},pt={json:"boolean","api-key":"string","base-url":"string"},ft={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},Ie={command:"help",positionals:[],flags:{}};function bt(e){if(e.length===0)throw new g("No command given");if(e[0]==="--help"||e[0]==="-h")return Ie;let t=e[0],r=ft[t];if(!r)throw new g(`Unknown command "${t}"`);let n={},s=[];for(let o=1;o<e.length;o++){let a=e[o];if(a==="--help"||a==="-h")return Ie;if(!a.startsWith("--")){s.push(a);continue}let i=a.slice(2),u,l=i.indexOf("=");l!==-1&&(u=i.slice(l+1),i=i.slice(0,l));let c=r.flags[i]??pt[i];if(!c)throw new g(`Unknown option --${i} for "${t}"`);if(c==="boolean"){if(u!==void 0)throw new g(`Option --${i} takes no value`);n[i]=true;continue}let d=u;if(d===void 0){let m=e[o+1];if(m===void 0||m.startsWith("--"))throw new g(`Option --${i} requires a value`);d=m,o++;}if(c==="number"){let m=Number(d);if(d.trim()===""||!Number.isFinite(m))throw new g(`Option --${i} expects a number, got "${d}"`);n[i]=m;}else if(c==="repeat"){let m=n[i]??[];m.push(d),n[i]=m;}else n[i]=d;}if(s.length<r.minPositionals)throw new g(`"${t}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(s.length>r.maxPositionals)throw new g(`"${t}" got unexpected argument "${s[r.maxPositionals]}"`);for(let o of r.required??[])if(!(o in n))throw new g(`"${t}" requires --${o}`);return {command:t,positionals:s,flags:n}}function ht(e){let t=e.indexOf(":");if(t<=0||t===e.length-1)throw new g(`Invalid --agent "${e}": expected harness:model[:version]`);let r=e.slice(0,t),n=e.slice(t+1),s=n.indexOf(":");if(s===-1)return {harness:r,model:n};let o=n.slice(0,s),a=n.slice(s+1);if(!o||!a)throw new g(`Invalid --agent "${e}": expected harness:model[:version]`);return {harness:r,model:o,harnessVersion:a}}function yt(e){let t=e.flags,r;if(t.tasks!==void 0&&(r=String(t.tasks).split(",").map(n=>n.trim()).filter(Boolean),r.length===0))throw new g("--tasks got an empty task list");return {benchmark:t.benchmark,...r!==void 0?{tasks:r}:{},agents:t.agent.map(n=>{let s=ht(n);return t.effort!==void 0?{...s,reasoningEffort:t.effort}:s}),...t.runs!==void 0?{runsPerTask:t.runs}:{},...t.concurrency!==void 0?{concurrency:t.concurrency}:{},...t["max-trial-spend"]!==void 0?{maxTrialSpendUsd:t["max-trial-spend"]}:{},...t.provider!==void 0?{sandboxProvider:t.provider}:{}}}function kt(e){let t=e.flags,r=typeof t.dir=="string",n=typeof t.git=="string"||typeof t.ref=="string";if(r&&n)throw new g('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let s of ["name","version"])if(typeof t[s]!="string")throw new g(`"import" requires --${s}`);return {source:{directory:t.dir},benchmarkName:t.name,version:t.version}}for(let s of ["git","ref","name","version"])if(typeof t[s]!="string"){let o=s==="git"||s==="ref"?" (or --dir for a local corpus directory)":"";throw new g(`"import" requires --${s}${o}`)}return {source:{gitUrl:t.git,ref:t.ref},benchmarkName:t.name,version:t.version}}function Rt(e){let t={};for(let r of e){let n=r.indexOf("=");if(n<=0)throw new g(`Invalid --env "${r}": expected KEY=VALUE`);t[r.slice(0,n)]=r.slice(n+1);}return t}function wt(e,t=r=>fs.readFileSync(r,"utf-8")){let r=e.flags,n=typeof r.dir=="string",s=typeof r["install-script"]=="string";if(n&&s)throw new g('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!s)throw new g('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let a of ["name","run"])if(typeof r[a]!="string")throw new g(`"custom-harnesses add" requires --${a}`);let o=Rt(r.env??[]);return {name:r.name,...n?{directory:r.dir}:{installScript:t(r["install-script"])},runCommand:r.run,...Object.keys(o).length>0?{env:o}:{}}}var vt={out:e=>process.stdout.write(e+`
|
|
86
86
|
`),err:e=>process.stderr.write(e+`
|
|
87
|
-
`)};function
|
|
88
|
-
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function
|
|
89
|
-
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function Dt(e,t){let n=await
|
|
90
|
-
`)
|
|
87
|
+
`)};function k(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join(" ").trimEnd())}function C(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function ae(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function L(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(ae).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",C(e.maxTrialSpendUsd)]),t.push(["worst case",C(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",C(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),k(t)}function St(e){return [e.id,e.status,e.benchmark,String(e.trials.total),D(e.meanReward),C(e.spentUsd),e.createdAt]}function It(e){return [e.taskKey,ae(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",C(e.spentUsd),e.id]}function xt(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",ae(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];if(e.metrics&&Object.keys(e.metrics).length>0&&t.push(["metrics",Object.entries(e.metrics).map(([r,n])=>`${r}=${n}`).join(" \xB7 ")]),t.push(["spent",C(e.spentUsd)]),(e.status==="RUNNING"||e.status==="SCORING")&&e.liveSpentUsd!==null){let r=e.liveSpendAt?` as of ${e.liveSpendAt}`:"";t.push(["spent (live)",`at least $${e.liveSpentUsd.toFixed(4)}${r}`]);}return e.sandboxProvider&&t.push(["provider",e.sandboxProvider]),e.sandboxId&&t.push(["sandbox",e.sandboxId]),e.verifierMode&&t.push(["verifier",e.verifierMode]),e.verifierSandboxId&&t.push(["verifier sandbox",e.verifierSandboxId]),e.resolvedHarnessVersion&&t.push(["harness version",e.resolvedHarnessVersion]),e.phaseTimingsMs&&Object.keys(e.phaseTimingsMs).length>0&&t.push(["timings",Object.entries(e.phaseTimingsMs).map(([r,n])=>`${r}=${n}ms`).join(" \xB7 ")]),e.failurePhase&&t.push(["failure phase",e.failurePhase]),e.failureDetail&&t.push(["failure detail",e.failureDetail]),e.sessionRef&&t.push(["session",e.sessionRef]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),k(t)}function Tt(e){if(e===null)return "-";let t=Math.round(e*1e3)/1e3;return t>0?`+${t}`:String(t)}function Ct(e){return [e.taskKey,e.status,D(e.sourceReward),D(e.reward),Tt(e.rewardDelta),e.sourceTrialId]}function Ce(e){let t=[["job id",e.id],["status",e.status],["source job",e.sourceJobId],["provider",e.sandboxProvider],["results",String(e.results.total)]],r=Object.entries(e.results.byStatus).filter(([,s])=>s>0).map(([s,o])=>`${s} ${o}`).join(" \xB7 ");if(r&&t.push(["by status",r]),e.filter&&(e.filter.status?.length||e.filter.taskKey)){let s=[];e.filter.status?.length&&s.push(`status=${e.filter.status.join(",")}`),e.filter.taskKey&&s.push(`task=${e.filter.taskKey}`),t.push(["filter",s.join(" \xB7 ")]);}t.push(["created",e.createdAt]);let n=k(t);if(e.results.items.length>0){n.push("");let s=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let o of e.results.items)s.push(Ct(o));n.push(...k(s)),e.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${e.id} --cursor ${e.results.nextCursor}`);}return n}function xe(e){let t=[["name",e.name],["source",e.source],["run command",e.runCommand]],r=Object.keys(e.env??{});return r.length>0&&t.push(["env",r.sort().join(", ")]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),k(t)}var Pe=["e2b","daytona","modal"];function Pt(e){return Pe.filter(t=>e?.[t]!==void 0).map(t=>`${t} ${e[t].ok?"ok":"NO"}`).join(" \xB7 ")}function D(e){return e!==null?String(Math.round(e*1e3)/1e3):"-"}function Et(e){let t=W(JSON.stringify(e.data??{}),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${t}`.trimEnd()}function Ee(e){let t=e.failures?.length?` (${e.failures.length} task failure${e.failures.length===1?"":"s"})`:"";return `${e.message}${t}`}function se(e){let t=[["id",e.id],["status",e.status]];if(e.benchmarkName!==void 0&&t.push(["benchmark",e.benchmarkName]),e.version!==void 0&&t.push(["version",e.version]),e.taskCount!==void 0&&t.push(["tasks",String(e.taskCount)]),e.failure){t.push(["failure",Ee(e.failure)]);for(let r of e.failure.failures??[])t.push([` ${r.taskKey}`,r.error]);}return k(t)}function At(e){let t=[];return e.taskCount!==void 0&&t.push(`tasks=${e.taskCount}`),e.failure&&t.push(W(Ee(e.failure),140)),`status ${e.status.padEnd(12)} ${t.join(" ")}`.trimEnd()}function Ot(e){let t={...e.data??{}},r=[];typeof t.trialId=="string"&&r.push(t.trialId),typeof t.taskKey=="string"&&r.push(t.taskKey),typeof t.status=="string"&&r.push(t.status),typeof t.reward=="number"&&r.push(`reward=${t.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[o,a]of Object.entries(t))n.has(o)||r.push(`${o}=${typeof a=="object"?JSON.stringify(a):String(a)}`);let s=W(r.join(" "),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${s}`.trimEnd()}function y(e){let t={};return typeof e.flags["api-key"]=="string"&&(t.apiKey=e.flags["api-key"]),typeof e.flags["base-url"]=="string"&&(t.baseUrl=e.flags["base-url"]),t}function G(e){return {...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{}}}function Jt(e){return e.status==="COMPLETED"?0:e.status==="FAILED"||e.status==="CANCELLED"?1:0}async function _t(e,t){let r=yt(e),n=e.flags.json===true,s=e.flags.watch===true,o=R(y(e)),a=await o.run(r);if(!s){if(n)t.out(JSON.stringify(a));else {for(let u of L(a))t.out(u);t.out(""),t.out(`Follow it with: evolve-evals get ${a.id}`);}return 0}n?t.out(JSON.stringify({kind:"job.created",job:a})):t.out(`Job ${a.id} (${a.benchmark}) ${a.status} \u2014 watching\u2026`);let i=await o.watch(a.id,{onEvent:u=>{t.out(n?JSON.stringify({kind:"event",...u}):Ot(u));}});if(n)t.out(JSON.stringify({kind:"job.final",job:i}));else {t.out("");for(let u of L(i))t.out(u);}return Jt(i)}async function Ut(e,t){let n=await R(y(e)).list({...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(n)),0;if(n.items.length===0)return t.out("No jobs."),0;let s=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let o of n.items)s.push(St(o));for(let o of k(s))t.out(o);return n.nextCursor&&t.out(`
|
|
88
|
+
More: evolve-evals list --cursor ${n.nextCursor}`),0}async function $t(e,t){let n=await R(y(e)).get(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of L(n))t.out(s);return 0}async function Lt(e,t){let r=R(y(e)),n;if(e.flags.status!==void 0&&(n=String(e.flags.status).split(",").map(a=>a.trim()).filter(Boolean),n.length===0))throw new g("--status got an empty status list");let s=await r.trials(e.positionals[0],{...n!==void 0?{status:n}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(s)),0;if(s.items.length===0)return t.out("No trials."),0;let o=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let a of s.items)o.push(It(a));for(let a of k(o))t.out(a);return t.out(`
|
|
89
|
+
${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function Dt(e,t){let n=await R(y(e)).trial(e.positionals[0],e.positionals[1]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of xt(n))t.out(s);return 0}async function Bt(e,t){let r=R(y(e)),n=e.flags.json===true,s=e.flags.stream;if(s!==void 0){if(s==="verifier"||s==="trace-stdout"||s==="trace-stderr"){let i=await r.trialArtifact(e.positionals[0],e.positionals[1],s);return i===null?(t.out(n?JSON.stringify({log:null}):`No ${s} log was stored for this trial.`),0):(t.out(n?JSON.stringify({log:i}):i),0)}if(s==="agent-home"){let i=await r.trialArtifact(e.positionals[0],e.positionals[1],s);if(i===null)return t.out(n?JSON.stringify({files:null}):`No ${s} content was stored for this trial.`),0;if(n)t.out(JSON.stringify({files:i}));else for(let[u,l]of Object.entries(i))t.out(`===== ${u} (${Buffer.byteLength(l,"utf8")} bytes) =====`),t.out(l);return 0}return t.err('--stream must be "verifier", "trace-stdout", "trace-stderr" or "agent-home"'),1}let o=e.flags.save;if(o!==void 0){let{mkdir:i,writeFile:u}=await import('fs/promises'),{join:l,dirname:c}=await import('path');await i(o,{recursive:true});let d=[];for await(let f of r.trialTraceEvents(e.positionals[0],e.positionals[1]))d.push(JSON.stringify(f));await u(l(o,"trace-parsed.jsonl"),d.join(`
|
|
90
|
+
`)+(d.length?`
|
|
91
|
+
`:"")),t.out(`trace-parsed.jsonl (${d.length} events)`);for(let f of ["verifier","trace-stdout","trace-stderr"]){let h=await r.trialArtifact(e.positionals[0],e.positionals[1],f);h!==null&&(await u(l(o,`${f}.log`),h),t.out(`${f}.log (${Buffer.byteLength(h,"utf8")} bytes)`));}let m=await r.trialArtifact(e.positionals[0],e.positionals[1],"agent-home");if(m!==null){for(let[f,h]of Object.entries(m)){let S=l(o,"agent-home",...f.split("/").filter(Boolean));await i(c(S),{recursive:true}),await u(S,h);}t.out(`agent-home/ (${Object.keys(m).length} files)`);}return 0}let a=0;for await(let i of r.trialTraceEvents(e.positionals[0],e.positionals[1],{...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{}}))t.out(n?JSON.stringify(i):Et(i)),a+=1;return !n&&a===0&&t.out("No trace events."),0}function Nt(e){let t=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of e.jobs)r.push([n.id,n.benchmark,n.status,D(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,C(n.spentUsd)]);if(t.push(...k(r)),e.taskMatrix.length>0){t.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...e.jobs.map((o,a)=>`JOB ${a+1}`)]],s=e.jobs.map(o=>o.id);for(let o of e.taskMatrix){let a=new Map(o.cells.map(i=>[i.jobId,i]));n.push([o.taskKey,o.disagreement?"!":"",...s.map(i=>{let u=a.get(i);return u?u.meanReward!==null?`${u.status} ${D(u.meanReward)}`:u.status:"-"})]);}t.push(...k(n));}return t}async function jt(e,t){let n=await R(y(e)).compare(e.positionals);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Nt(n))t.out(s);return 0}async function Ht(e,t){let n=await R(y(e)).cancel(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of L(n))t.out(s);return 0}async function Mt(e,t){let n=await R(y(e)).rerunFailed(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else {for(let s of L(n))t.out(s);t.out(""),t.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function Vt(e,t){let r=R(y(e)),[n,s]=e.positionals,o;if(s!==void 0){if(e.flags.status!==void 0||e.flags.task!==void 0)throw new g("--status/--task apply to a whole-job regrade, not a single trial");o=await r.regradeTrial(n,s);}else {let a={};if(e.flags.status!==void 0){let i=String(e.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(i.length===0)throw new g("--status got an empty status list");a.status=i;}e.flags.task!==void 0&&(a.taskKey=String(e.flags.task)),o=await r.regrade(n,a);}if(e.flags.json===true)t.out(JSON.stringify(o));else {for(let a of Ce(o))t.out(a);t.out(""),t.out(`Follow it with: evolve-evals regrade-job ${o.id}`);}return 0}async function Kt(e,t){let n=await R(y(e)).getRegrade(e.positionals[0],G(e));if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Ce(n))t.out(s);return 0}async function Ft(e,t){let r=e.flags.format;if(r!==void 0&&r!=="harbor")throw new g(`Unknown --format "${r}" (supported: harbor)`);let s=await R(y(e)).export(e.positionals[0],{to:e.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return e.flags.json===true?t.out(JSON.stringify({path:s})):t.out(`Saved ${s}`),0}async function qt(e,t){let n=await q(y(e)).downloadPackage(e.positionals[0],{to:e.flags.to??process.cwd()});return e.flags.json===true?t.out(JSON.stringify({path:n})):t.out(`Saved ${n}`),0}function Gt(e){let t=k([["name",e.name],["title",e.title??"-"],["description",e.description??"-"],["active version",e.activeVersion?.version??"-"]]);if(e.versions&&e.versions.length>0){t.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let n of e.versions)r.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);t.push(...k(r));}if(e.tasks&&e.tasks.items.length>0){t.push("",`Tasks (version ${e.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let s of e.tasks.items)r.push([s.taskKey,`${s.agentTimeoutSec}s`,`${s.verifierTimeoutSec}s`,Pt(s.providers)]);t.push(...k(r)),e.tasks.nextCursor&&t.push(`More tasks: evolve-evals benchmarks get ${e.name} --cursor ${e.tasks.nextCursor}`);let n=new Map;for(let s of e.tasks.items)for(let o of Pe){let a=s.providers?.[o];a&&!a.ok&&!n.has(`${o}:${a.reason}`)&&n.set(`${o}:${a.reason}`,`${o}: ${a.reason}`);}if(n.size>0){t.push("","Provider limitations:");for(let s of n.values())t.push(` ${s}`);}}return t}async function Wt(e,t){let r=q(y(e)),[n,s]=e.positionals;if(n===void 0){let a=await r.list(G(e));if(e.flags.json===true)return t.out(JSON.stringify(a)),0;if(a.items.length===0)return t.out("No benchmarks."),0;let i=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let u of a.items)i.push([u.name,u.activeVersion?.version??"-",u.activeVersion?.state??"-",u.activeVersion?String(u.activeVersion.taskCount):"-",u.title??"-"]);for(let u of k(i))t.out(u);for(let u of Te(a.items))t.out(u);return 0}if(n!=="get")throw new g(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!s)throw new g('"benchmarks get" requires a <name[@version]> ref');let o=await r.get(s,G(e));if(e.flags.json===true)t.out(JSON.stringify(o));else {for(let a of Gt(o))t.out(a);for(let a of Te([o]))t.out(a);}return 0}function Te(e){let t=[];for(let r of e){if(!r.upstream?.moved)continue;let n=r.activeVersion?`@${r.activeVersion.version}`:"";t.push(`${r.name}${n} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return t}async function Yt(e,t){let[r,n]=e.positionals,s=e.flags.json===true,o=q(y(e));if(r==="status"){if(!n)throw new g('"import status" requires an <id>');let l=await o.getImport(n);if(s)t.out(JSON.stringify(l));else for(let c of se(l))t.out(c);return 0}if(r!==void 0)throw new g(`Unknown import subcommand "${r}" (supported: status)`);let a=kt(e),i=await o.import(a);if(e.flags.watch!==true){if(s)t.out(JSON.stringify(i));else {for(let l of se(i))t.out(l);t.out(""),t.out(`Follow it with: evolve-evals import status ${i.id}`);}return 0}s?t.out(JSON.stringify({kind:"import.created",benchmarkImport:i})):t.out(`Import ${i.id} (${a.benchmarkName}) ${i.status} \u2014 watching\u2026`);let u=await o.watchImport(i.id,{onStatus:l=>{t.out(s?JSON.stringify({kind:"import.status",benchmarkImport:l}):At(l));}});if(s)t.out(JSON.stringify({kind:"import.final",benchmarkImport:u}));else {t.out("");for(let l of se(u))t.out(l);}return u.status==="FAILED"?1:0}async function zt(e,t){let r=Se(y(e)),[n,s]=e.positionals,o=e.flags.json===true;if(n===void 0){let a=await r.list(G(e));if(o)return t.out(JSON.stringify(a)),0;if(a.items.length===0)return t.out("No custom harnesses."),0;let i=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let u of a.items)i.push([u.name,u.source,W(u.runCommand,60),u.updatedAt]);for(let u of k(i))t.out(u);return 0}if(n==="add"){if(s!==void 0)throw new g(`"custom-harnesses add" got unexpected argument "${s}"`);let a=await r.create(wt(e));if(o)t.out(JSON.stringify(a));else {for(let i of xe(a))t.out(i);t.out(""),t.out(`Use it with: evolve-evals run --agent ${a.name}:<model> \u2026`);}return 0}if(n==="get"){if(!s)throw new g('"custom-harnesses get" requires a <name>');let a=await r.get(s);if(o)t.out(JSON.stringify(a));else for(let i of xe(a))t.out(i);return 0}if(n==="remove"){if(!s)throw new g('"custom-harnesses remove" requires a <name>');return await r.delete(s),o?t.out(JSON.stringify({name:s,deleted:true})):t.out(`Deleted custom harness ${s}`),0}throw new g(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function Qt(e,t=vt){let r;try{r=bt(e);}catch(n){if(n instanceof g)return t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(r.command){case "help":return t.out(gt),0;case "run":return await _t(r,t);case "list":return await Ut(r,t);case "get":return await $t(r,t);case "trials":return await Lt(r,t);case "trial":return await Dt(r,t);case "trace":return await Bt(r,t);case "compare":return await jt(r,t);case "cancel":return await Ht(r,t);case "rerun-failed":return await Mt(r,t);case "regrade":return await Vt(r,t);case "regrade-job":return await Kt(r,t);case "export":return await Ft(r,t);case "benchmarks":return await Wt(r,t);case "import":return await Yt(r,t);case "download":return await qt(r,t);case "custom-harnesses":return await zt(r,t);default:return t.err(`Error: unknown command "${r.command}"`),2}}catch(n){return n instanceof g?(t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2):(t.err(`Error: ${n.message}`),1)}}var Xt=(()=>{let e=process.argv[1];if(!e)return false;let t=r=>{try{return (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('cli.cjs', document.baseURI).href))===url.pathToFileURL(r).href}catch{return false}};if(t(e))return true;try{return t(fs.realpathSync(e))}catch{return false}})();Xt&&Qt(process.argv.slice(2)).then(e=>{process.exitCode=e;},e=>{process.stderr.write(`Error: ${e?.message??e}
|
|
92
|
+
`),process.exitCode=1;});exports.CliUsageError=g;exports.USAGE=gt;exports.buildCustomHarnessInput=wt;exports.buildImportInput=kt;exports.buildJobInput=yt;exports.eventLine=Ot;exports.importStatusLine=At;exports.parseArgs=bt;exports.parseHarnessEnv=Rt;exports.parseJobAgent=ht;exports.runCli=Qt;exports.traceEventLine=Et;exports.trialDetailLines=xt;
|
package/dist/hosted/cli.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-
|
|
2
|
+
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, t as TrialDetail, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BBerSem-.cjs';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* evolve-evals — CLI for Evolve hosted benchmarks & jobs.
|
|
@@ -12,7 +12,7 @@ import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHa
|
|
|
12
12
|
* runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n download <import-id> Download the original corpus package (owner only)\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir> Directory to save into, for export and download (default: current dir)\n --format harbor Export the Harbor job-layout bundle\n --json Machine-readable JSON output\n --api-key <key> API key (default: $EVOLVE_API_KEY)\n --base-url <url> API base URL (default: the Evolve dashboard API)";
|
|
15
|
+
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events (--stream <raw-artifact>, --save <dir> for all)\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n download <import-id> Download the original corpus package (owner only)\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir> Directory to save into, for export and download (default: current dir)\n --format harbor Export the Harbor job-layout bundle\n --json Machine-readable JSON output\n --api-key <key> API key (default: $EVOLVE_API_KEY)\n --base-url <url> API base URL (default: the Evolve dashboard API)";
|
|
16
16
|
/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
|
|
17
17
|
declare class CliUsageError extends Error {
|
|
18
18
|
constructor(message: string);
|
|
@@ -45,6 +45,11 @@ interface CliIO {
|
|
|
45
45
|
out(line: string): void;
|
|
46
46
|
err(line: string): void;
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Full-detail rendering of one trial — evolve-evals trial. Exported for tests,
|
|
50
|
+
* like the other line renderers.
|
|
51
|
+
*/
|
|
52
|
+
declare function trialDetailLines(run: TrialDetail): string[];
|
|
48
53
|
/** One trace event line — evolve-evals trace. */
|
|
49
54
|
declare function traceEventLine(event: TrialTraceEvent): string;
|
|
50
55
|
/** Compact one-line rendering of one import status change for --watch. */
|
|
@@ -53,4 +58,4 @@ declare function importStatusLine(job: BenchmarkImport): string;
|
|
|
53
58
|
declare function eventLine(event: JobEvent): string;
|
|
54
59
|
declare function runCli(argv: string[], io?: CliIO): Promise<number>;
|
|
55
60
|
|
|
56
|
-
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };
|
|
61
|
+
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine, trialDetailLines };
|
package/dist/hosted/cli.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-
|
|
2
|
+
import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, t as TrialDetail, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BBerSem-.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* evolve-evals — CLI for Evolve hosted benchmarks & jobs.
|
|
@@ -12,7 +12,7 @@ import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHa
|
|
|
12
12
|
* runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n download <import-id> Download the original corpus package (owner only)\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir> Directory to save into, for export and download (default: current dir)\n --format harbor Export the Harbor job-layout bundle\n --json Machine-readable JSON output\n --api-key <key> API key (default: $EVOLVE_API_KEY)\n --base-url <url> API base URL (default: the Evolve dashboard API)";
|
|
15
|
+
declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n run Create a job (add --watch to follow it)\n list List your jobs (newest first)\n get <id> Show one job\n trials <id> List a job's trials\n trial <id> <trial-id> Show one trial in full detail\n trace <id> <trial-id> Print a trial's trace events (--stream <raw-artifact>, --save <dir> for all)\n compare <id> <id> [...] Compare 2-5 jobs side by side\n cancel <id> Request cancellation of a job\n rerun-failed <id> New job from a terminal job's failed trials\n regrade <id> [trial-id] Re-run the verifier on recorded trials (whole job, or one trial)\n regrade-job <job-id> Show a regrade job's results (rewards, deltas, lineage)\n export <id> Download the research archive (gzipped JSON)\n benchmarks List the benchmark catalog\n benchmarks get <name[@version]> Show one benchmark (versions + tasks + providers)\n import Import a benchmark from a git source or a local directory (--watch to follow)\n import status <id> Show one import job\n download <import-id> Download the original corpus package (owner only)\n custom-harnesses List your registered custom harnesses\n custom-harnesses get <name> Show one custom harness\n custom-harnesses add Register a custom harness (install script or local directory)\n custom-harnesses remove <name> Delete a custom harness\n help Show this help\n\nRun options:\n --benchmark <name[@version]> Benchmark (required; bare name = active version)\n --tasks <k1,k2,...> Task keys (default: every task of the version)\n --agent <harness:model[:version]> Agent; repeatable (at least one required)\n --effort <value> Reasoning effort for EVERY agent (values:\n GET /api/meta limits.job.reasoningEfforts).\n Applied verbatim \u2014 an agent whose harness\n cannot honor it is refused by the server,\n never silently skipped. Per-agent efforts\n need the SDK. Omitted: the server default.\n --runs <n> Runs per task x agent (default 1)\n --concurrency <n> Parallel trials (default 1)\n --max-trial-spend <usd> Model-spend cap for EACH trial (default: the server's, $200)\n --provider <e2b|daytona|modal> e2b | daytona | modal, default e2b\n --watch Stream events until the job finishes\n\nTrial options:\n --status <s1,s2,...> Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n --status <s1,s2,...> Only regrade source trials in these statuses\n --task <key> Only regrade source trials of this task\n\nTrace options:\n --cursor <seq> Resume after this trace seq (a trace cursor IS a seq)\n --limit <n> Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n --git <url> Git repository URL (with --ref)\n --ref <ref> Git ref: branch, tag, or commit (with --git)\n --dir <path> Local corpus directory (tarred + uploaded)\n --name <benchmark> Catalog benchmark name to create or extend (required)\n --version <v> Version label for the imported version (required)\n --watch Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n --name <harness> Harness name, later used in --agent (required)\n --install-script <path> Install script file; its contents are uploaded\n --dir <path> Local harness directory (tarred + uploaded)\n --run <command> Run command, executed with sh -c (required)\n --env KEY=VALUE Env injected at run time; repeatable\n\nOther options:\n --limit <n>, --cursor <c> Pagination \u2014 one envelope on every collection\n (list, trials, trace, benchmarks, benchmarks get,\n custom-harnesses, regrade-job)\n --to <dir> Directory to save into, for export and download (default: current dir)\n --format harbor Export the Harbor job-layout bundle\n --json Machine-readable JSON output\n --api-key <key> API key (default: $EVOLVE_API_KEY)\n --base-url <url> API base URL (default: the Evolve dashboard API)";
|
|
16
16
|
/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
|
|
17
17
|
declare class CliUsageError extends Error {
|
|
18
18
|
constructor(message: string);
|
|
@@ -45,6 +45,11 @@ interface CliIO {
|
|
|
45
45
|
out(line: string): void;
|
|
46
46
|
err(line: string): void;
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Full-detail rendering of one trial — evolve-evals trial. Exported for tests,
|
|
50
|
+
* like the other line renderers.
|
|
51
|
+
*/
|
|
52
|
+
declare function trialDetailLines(run: TrialDetail): string[];
|
|
48
53
|
/** One trace event line — evolve-evals trace. */
|
|
49
54
|
declare function traceEventLine(event: TrialTraceEvent): string;
|
|
50
55
|
/** Compact one-line rendering of one import status change for --watch. */
|
|
@@ -53,4 +58,4 @@ declare function importStatusLine(job: BenchmarkImport): string;
|
|
|
53
58
|
declare function eventLine(event: JobEvent): string;
|
|
54
59
|
declare function runCli(argv: string[], io?: CliIO): Promise<number>;
|
|
55
60
|
|
|
56
|
-
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };
|
|
61
|
+
export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine, trialDetailLines };
|