@evolvingmachines/sdk 0.0.51 → 0.0.52-project-sable.20260729.8632c49

package/dist/chunk-ZYTUS4TY.js

@@ -0,0 +1,6 @@
+import {createWriteStream}from'fs';import {mkdir,rename,rm}from'fs/promises';import {join,basename}from'path';import {Readable}from'stream';import {pipeline}from'stream/promises';function le(e=""){let t=process.env.EVOLVE_GATEWAY_URL||"https://swarmkit-gateway-692833842999.us-central1.run.app";if(!e)return t;let n=e.startsWith("/")?e:`/${e}`;return `${t}${n}`}function de(e=""){let t=process.env.EVOLVE_DASHBOARD_URL||w;if(!e)return t;let n=e.startsWith("/")?e:`/${e}`;return `${t.replace(/\/$/,"")}${n}`}var Ue=["e2b","daytona","modal"];function W(e){return de(`/api/managed/${e}`)}function He(){return `${W("daytona")}/toolbox`}function Me(){return W("e2b")}function Ne(e){return {"browser-use":{type:"http",url:le("/browser_use/mcp"),headers:{"x-litellm-api-key":`Bearer ${e}`}}}}var M="EVOLVE_API_KEY",je="E2B_API_KEY",Ve="DAYTONA_API_KEY",$e="MODAL_TOKEN_ID",Ke="MODAL_TOKEN_SECRET",Fe="claude",Ge=36e5,qe=4,We=100,Y="/home/user";function Ye(e,t){return t?t.replace(/\/+$/,"")||"/":e?e==="root"?"/root":`/home/${e}`:Y}var w=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai",Qe="x-litellm-customer-id",Xe="x-litellm-tags",Ze="run:",et=".evolve-sdk/observability/sessions",tt=100,rt=5e3,nt=3,st=1e3;var z=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","upstream_not_watchable","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","package_not_retained","package_corrupt","package_missing","internal_error"];function N(e){return typeof e=="string"&&z.includes(e)}var A=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,n,s,o){super(s),this.name="EvolveApiError",this.status=t,this.code=n,o?.param!==void 0&&(this.param=o.param),o?.details!==void 0&&(this.details=o.details),o?.retryAfterSec!==void 0&&(this.retryAfterSec=o.retryAfterSec),o?.requestId!==void 0&&(this.requestId=o.requestId);}isKnownCode(){return N(this.code)}};async function $(e){let t=await e.text().catch(()=>""),n=e.headers?.get?.("x-request-id")??void 0,s=Number(e.headers?.get?.("retry-after")),o=Number.isFinite(s)?s:void 0;try{let d=JSON.parse(t);if(d?.error&&typeof d.error=="object"){let l=typeof d.error.code=="string"?d.error.code:"unknown_error",c=typeof d.error.message=="string"?d.error.message:e.statusText;throw new A(e.status,l,c,{param:typeof d.error.param=="string"?d.error.param:void 0,details:d.error.details&&typeof d.error.details=="object"?d.error.details:void 0,retryAfterSec:typeof d.error.retryAfterSec=="number"?d.error.retryAfterSec:o,requestId:typeof d.error.requestId=="string"?d.error.requestId:n})}}catch(d){if(d instanceof A)throw d}throw new A(e.status,"unknown_error",t||e.statusText,{retryAfterSec:o,requestId:n})}var j=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},be=1e3,fe=3e4,he=new Set(["COMPLETED","CANCELLED","FAILED"]),Re=new Set(["job.completed","job.cancelled","job.failed"]),ye=2e3,ke=new Set(["COMPLETED","FAILED"]);function K(e,t){let n=t?.apiKey||process.env[M];if(!n)throw new Error(`${e}() requires an API key. Set ${M} or pass { apiKey } in config.`);let s=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||w).replace(/\/$/,"");return {apiKey:n,baseUrl:s}}async function m(e,t,n){let s=await fetch(`${e.baseUrl}${t}`,{...n,headers:{Authorization:`Bearer ${e.apiKey}`,...n?.headers}});return s.ok||await $(s),s}function ve(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let n=e.slice(0,t).trim(),s=e.slice(t+1).trim();if(!n||!s)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:n,version:s}}function F(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function _(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function Te(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function I(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(F),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function k(e,t){let n=e??{};return {items:Array.isArray(n.items)?n.items.map(t):[],nextCursor:n.nextCursor??null,hasMore:n.hasMore===true}}function y(e,t){let n=new URLSearchParams;e?.limit!==void 0&&n.set("limit",String(e.limit)),e?.cursor&&n.set("cursor",e.cursor);for(let[o,d]of Object.entries(t??{}))d!==void 0&&n.set(o,d);let s=n.toString();return s?`?${s}`:""}function Q(e){return {id:e.id,taskKey:e.taskKey,agent:F(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function Ce(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function O(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...k(t,Ce),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function J(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function B(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function ae(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function xe(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:ae(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(F),createdAt:e.createdAt}}function Se(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:ae(e.coverage)}}function Pe(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(Se)}}function Ee(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function V(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function X(e,t){return t?.aborted?true:e?.name==="AbortError"}function D(e,t){return new Promise((n,s)=>{let o=setTimeout(()=>{t?.removeEventListener("abort",d),n();},e),d=()=>{clearTimeout(o),s(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",d,{once:true});})}function Ie(e){let t="";return {push(n){t+=n.replace(/\r\n/g,`
+`);let s;for(;(s=t.indexOf(`
+
+`))!==-1;){let o=t.slice(0,s);t=t.slice(s+2);let d,l,c=[];for(let r of o.split(`
+`)){if(!r||r.startsWith(":"))continue;let a=r.indexOf(":"),u=a===-1?r:r.slice(0,a),i=a===-1?"":r.slice(a+1);i.startsWith(" ")&&(i=i.slice(1)),u==="id"?d=i:u==="event"?l=i:u==="data"&&c.push(i);}(d!==void 0||l!==void 0||c.length>0)&&e({id:d,event:l,data:c.join(`
+`)});}}}}function P(e,t){let n,s=()=>n??=e({limit:t?.limit,cursor:t?.cursor});return {then(o,d){return s().then(o,d)},catch(o){return s().catch(o)},finally(o){return s().finally(o)},async*[Symbol.asyncIterator](){let o=t?.cursor;for(;;){let d=await e({limit:t?.limit,cursor:o});for(let l of d.items)yield l;if(!d.nextCursor)return;o=d.nextCursor;}}}}function L(e,t){let n=new FormData;for(let[s,o]of Object.entries(e))o!==void 0&&n.set(s,o);return t&&n.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),n}function Ae(e){let t,n=()=>(t||(t=(async()=>{let s=await e.next();for(;!s.done;)s=await e.next();return s.value})()),t);return {then(s,o){return n().then(s,o)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},[Symbol.asyncIterator](){return e}}}var oe="x-package-sha256",U=class extends Error{constructor(n,s){super(`downloaded bytes do not match the digest the server stated (expected ${n}, got ${s})`);this.expected=n;this.actual=s;}name="EvolveDigestMismatchError"},H=class extends Error{constructor(n,s){super(`download ended early: the server declared ${n} bytes and ${s} arrived`);this.expectedBytes=n;this.receivedBytes=s;}name="EvolveIncompleteDownloadError"};function we(e,t){let n=e.headers.get("Content-Length");if(n===null)return;let s=Number(n);if(Number.isFinite(s)&&t!==s)throw new H(s,t)}async function _e(e,t){let n=e.headers.get(oe);if(!n)return;let{createHash:s}=await import('crypto'),o=s("sha256").update(t).digest("hex");if(o!==n)throw new U(n,o)}function ie(e,t){let s=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!s)return t;let o=basename(s[1]);return o===""||o==="."||o===".."||/[/\\]/.test(o)||/[\u0000-\u001f]/.test(o)?t:o}function Oe(e){let t=K("benchmarks",e);async function n(l){let c=await m(t,`/api/benchmarks/imports/${encodeURIComponent(l)}`);return B(await c.json())}function s(l){if(!l||typeof l!="object")return null;let c=l;return {ref:c.ref,currentCommit:c.currentCommit,latestCommit:c.latestCommit??null,moved:c.moved===true,behindBy:typeof c.behindBy=="number"?c.behindBy:null,checkedAt:c.checkedAt??null,error:c.error??null}}async function o(l,c){let r=ve(l),a=y(c,{version:r.version}),i=await(await m(t,`/api/benchmarks/${encodeURIComponent(r.name)}${a}`)).json();return {name:i.name,title:i.title??null,description:i.description??null,activeVersion:i.activeVersion?_(i.activeVersion):null,versions:(i.versions||[]).map(_),selectedVersion:i.selectedVersion?_(i.selectedVersion):null,tasks:k(i.tasks,Te),upstream:s(i.upstream),createdAt:i.createdAt,updatedAt:i.updatedAt}}async function d(l){let c=await m(t,`/api/benchmarks${y(l)}`);return k(await c.json(),r=>({name:r.name,title:r.title??null,description:r.description??null,activeVersion:r.activeVersion?_(r.activeVersion):null,upstream:s(r.upstream)}))}return {list(l){return P(d,l)},get:o,async getActive(l,c){let r=await o(l,c);if(r.activeVersion===null)throw new j(l);return {name:r.name,title:r.title,description:r.description,activeVersion:r.activeVersion,version:r.activeVersion.version,tasks:r.tasks??{items:[],nextCursor:null,hasMore:false},versions:r.versions??[],createdAt:r.createdAt,updatedAt:r.updatedAt}},async import(l){let c=l.source;if(c?.directory){let{tarGzipDirectory:r}=await import('./tar-WPIXS3E6.js'),a=r(c.directory),u=await m(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:l.benchmarkName,version:l.version},{bytes:a,filename:"corpus.tar.gz"})});return B(await u.json())}if(c&&"gitUrl"in c&&c.gitUrl){let r=await m(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:l.benchmarkName,version:l.version,gitUrl:c.gitUrl,ref:c.ref})});return B(await r.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:n,async watchImport(l,c){let r=c?.pollIntervalMs??ye,a=null;for(;;){V(c?.signal);let u=await n(l);if(u.status!==a&&(a=u.status,c?.onStatus?.(u)),ke.has(u.status))return u;await D(r,c?.signal);}},downloadPackage:(async(l,c)=>{let r=await m(t,`/api/benchmarks/imports/${encodeURIComponent(l)}/package`);if(c?.stream){if(!r.body)throw new Error("Package response has no body");return r.body}if(c?.to){if(!r.body)throw new Error("Package response has no body");let u=c.to;await mkdir(u,{recursive:true});let i=join(u,ie(r,`import-${l}-corpus.tar.gz`)),{createHash:f,randomBytes:h}=await import('crypto'),R=`${i}.${h(8).toString("hex")}.part`,v=f("sha256"),T=0,p=Readable.fromWeb(r.body);p.on("data",g=>{v.update(g),T+=g.length;});try{await pipeline(p,createWriteStream(R));let g=r.headers.get("Content-Length");if(g!==null&&T!==Number(g))throw new H(Number(g),T);let C=r.headers.get(oe),x=v.digest("hex");if(C&&x!==C)throw new U(C,x);await rename(R,i);}catch(g){throw await rm(R,{force:true}).catch(()=>{}),g}return i}let a=Buffer.from(await r.arrayBuffer());return we(r,a.length),await _e(r,a),a}),listImports(l){return P(async c=>{let r=new URLSearchParams;c.limit!==void 0&&r.set("limit",String(c.limit)),c.cursor!==void 0&&r.set("cursor",c.cursor),l?.status!==void 0&&r.set("status",l.status),l?.benchmark!==void 0&&r.set("benchmark",l.benchmark);let a=r.toString()?`?${r}`:"",u=await m(t,`/api/benchmarks/imports${a}`);return k(await u.json(),B)},l)},async delete(l){await m(t,`/api/benchmarks/${encodeURIComponent(l)}`,{method:"DELETE"});}}}function Je(e){let t=K("customHarnesses",e);return {async create(n){let s=await Z("customHarnesses().create()",n),o=await m(t,"/api/custom-harnesses",{method:"POST",body:s});return J(await o.json())},list(n){return P(async s=>{let o=await m(t,`/api/custom-harnesses${y(s)}`);return k(await o.json(),J)},n)},async get(n){let s=await m(t,`/api/custom-harnesses/${encodeURIComponent(n)}`);return J(await s.json())},async delete(n){await m(t,`/api/custom-harnesses/${encodeURIComponent(n)}`,{method:"DELETE"});},async upsert(n,s){let o=await Z("customHarnesses().upsert()",{...s,name:n}),d=await m(t,`/api/custom-harnesses/${encodeURIComponent(n)}`,{method:"PUT",body:o});return J(await d.json())}}}async function Z(e,t){let n=typeof t.installScript=="string",s=typeof t.directory=="string";if(n&&s)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!n&&!s)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let o={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...n?{installScript:t.installScript}:{}};if(s){let{tarGzipDirectory:d}=await import('./tar-WPIXS3E6.js'),l=d(t.directory);return L(o,{bytes:l,filename:"source.tar.gz"})}return L(o)}function Be(e){let t=K("jobs",e);async function n(r){let a=await m(t,`/api/jobs/${encodeURIComponent(r)}`);return I(await a.json())}async function s(r){let a=await m(t,`/api/jobs${y(r)}`);return k(await a.json(),I)}async function o(r,a){let u=y(a,{status:a?.status?.length?a.status.join(","):void 0}),i=await m(t,`/api/jobs/${encodeURIComponent(r)}/trials${u}`);return k(await i.json(),Q)}async function d(r,a,u){let i=await m(t,`/api/jobs/${encodeURIComponent(r)}/trials/${encodeURIComponent(a)}/trace${y(u)}`);return k(await i.json(),Ee)}async function l(r,a){let u=a?`?format=${encodeURIComponent(a)}`:"";return m(t,`/api/jobs/${encodeURIComponent(r)}/export${u}`)}async function*c(r,a){let{onEvent:u,signal:i}=a??{},f=a?.reconnectDelayMs??be,h=a?.maxReconnectDelayMs??fe,R=null,v=false,T=false,p=f;for(;!v;){V(i);let g;try{g=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(r)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...R!==null?{"Last-Event-ID":String(R)}:{}},signal:i});}catch(b){if(X(b,i))throw b;await D(p,i),p=Math.min(p*2,h);continue}if(!g.ok){if(g.status===429||g.status>=500){await g.text().catch(()=>""),await D(p,i),p=Math.min(p*2,h);continue}await $(g);}if(!g.body)throw new Error("Event stream response has no body");let C=false,x=[],ue=Ie(b=>{let S=Number(b.id),E={seq:Number.isInteger(S)?S:-1,type:b.event||"message",data:b.data?De(b.data):{}};Number.isInteger(S)&&(R=S),C=true,Re.has(E.type)&&(v=true),x.push(E);}),G=g.body.getReader(),ce=new TextDecoder;try{for(;!v;){let{done:b,value:S}=await G.read();if(b)break;ue.push(ce.decode(S,{stream:!0}));for(let E of x)u?.(E),yield E;x.length=0;}}catch(b){if(X(b,i))throw b}finally{await G.cancel().catch(()=>{});}if(v)break;V(i);let q=await n(r);if(he.has(q.status)){if(T)return q;T=true;continue}C&&(p=f),await D(p,i),p=Math.min(p*2,h);}return n(r)}return {async run(r,a){let u=await m(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...a?.idempotencyKey?{"Idempotency-Key":a.idempotencyKey}:{}},body:JSON.stringify(r)});return I(await u.json())},get:n,list(r){return P(s,r)},trials(r,a){return P(u=>o(r,{...u,status:a?.status}),a)},async trial(r,a){let i=await(await m(t,`/api/jobs/${encodeURIComponent(r)}/trials/${encodeURIComponent(a)}`)).json();return {...Q(i),jobId:i.jobId}},trialTrace:d,async*trialTraceEvents(r,a,u){let i=u?.cursor;for(;;){let f=await d(r,a,{cursor:i,limit:u?.limit});for(let h of f.items)yield h;if(!f.nextCursor)return;i=f.nextCursor;}},watch(r,a){return Ae(c(r,a))},async cancel(r){let a=await m(t,`/api/jobs/${encodeURIComponent(r)}/cancel`,{method:"POST"});return I(await a.json())},async compare(r){let a=r.map(encodeURIComponent).join(","),i=await(await m(t,`/api/jobs/compare?ids=${a}`)).json();return {jobs:(i.jobs||[]).map(xe),taskMatrix:(i.taskMatrix||[]).map(Pe)}},async rerunFailed(r,a){let u=await m(t,`/api/jobs/${encodeURIComponent(r)}/rerun-failed`,{method:"POST",headers:a?.idempotencyKey?{"Idempotency-Key":a.idempotencyKey}:void 0});return I(await u.json())},async regrade(r,a){let u={};a?.status?.length&&(u.status=a.status),a?.taskKey!==void 0&&(u.taskKey=a.taskKey);let i=await m(t,`/api/jobs/${encodeURIComponent(r)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(u)});return O(await i.json())},async regradeTrial(r,a){let u=await m(t,`/api/jobs/${encodeURIComponent(r)}/trials/${encodeURIComponent(a)}/regrade`,{method:"POST"});return O(await u.json())},listRegrades(r){return P(async a=>{let i=await(await m(t,`/api/regrades${y(a,{jobId:r?.jobId})}`)).json();return {items:i.items.map(O),nextCursor:i.nextCursor,hasMore:i.hasMore}},r)},async getRegrade(r,a){let u=await m(t,`/api/regrades/${encodeURIComponent(r)}${y(a)}`);return O(await u.json())},export:(async(r,a)=>{let u=await l(r,a?.format);if(a?.stream){if(!u.body)throw new Error("Export response has no body");return u.body}if(a?.to){if(!u.body)throw new Error("Export response has no body");let f=a.to;await mkdir(f,{recursive:true});let h=join(f,ie(u,`job-${r}-export.json.gz`)),R=Readable.fromWeb(u.body);return await pipeline(R,createWriteStream(h)),h}let i=await u.arrayBuffer();return Buffer.from(i)})}}function De(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}function pt(e){let t,n,s;return {get benchmarks(){return t??=Oe(e)},get customHarnesses(){return n??=Je(e)},get jobs(){return s??=Be(e)},meta(){return Le(e)}}}async function Le(e){let t=(e?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||w).replace(/\/$/,""),n=await fetch(`${t}/api/meta`);return n.ok||await $(n),await n.json()}export{st as A,z as B,N as C,A as D,j as E,U as F,H as G,Oe as H,Je as I,Be as J,pt as K,Le as L,le as a,de as b,Ue as c,W as d,He as e,Me as f,Ne as g,M as h,je as i,Ve as j,$e as k,Ke as l,Fe as m,Ge as n,qe as o,We as p,Y as q,Ye as r,w as s,Qe as t,Xe as u,Ze as v,et as w,tt as x,rt as y,nt as z};

package/dist/hosted/cli.cjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+'use strict';var zlib=require('zlib'),fs=require('fs'),path=require('path'),url=require('url'),promises=require('fs/promises'),stream=require('stream'),promises$1=require('stream/promises');var _documentCurrentScript=typeof document!=='undefined'?document.currentScript:null;var Oe=Object.defineProperty;var Je=(e,t)=>()=>(e&&(t=e(e=0)),t);var _e=(e,t)=>{for(var r in t)Oe(e,r,{get:t[r],enumerable:true});};var Q={};_e(Q,{tarGzipDirectory:()=>He});function Ne(e){let t=[],r=n=>{let s=n===""?e:path.join(e,n),i=fs.readdirSync(s,{withFileTypes:true}).sort((o,u)=>o.name<u.name?-1:o.name>u.name?1:0);for(let o of i){if(o.name.startsWith("."))continue;let u=n===""?o.name:`${n}/${o.name}`,a=path.join(s,o.name),l=fs.lstatSync(a);l.isSymbolicLink()||(l.isDirectory()?r(u):l.isFile()&&t.push({rel:u,abs:a}));}};return r(""),t.sort((n,s)=>n.rel<s.rel?-1:n.rel>s.rel?1:0)}function je(e,t){let r=e,n="";if(Buffer.byteLength(r,"utf8")>100){let o=-1;for(let u=r.indexOf("/");u!==-1;u=r.indexOf("/",u+1)){let a=r.slice(0,u),l=r.slice(u+1);l.length>0&&Buffer.byteLength(l,"utf8")<=100&&Buffer.byteLength(a,"utf8")<=155&&(o=u);}if(o===-1)throw new Error(`path too long to store in a USTAR tar (max 255 bytes, split at "/"): ${e}`);n=r.slice(0,o),r=r.slice(o+1);}let s=Buffer.alloc(O);s.write(r,0,100,"utf8"),s.write("0000644\0",100,"ascii"),s.write("0000000\0",108,"ascii"),s.write("0000000\0",116,"ascii"),s.write(t.toString(8).padStart(11,"0")+"\0",124,"ascii"),s.write("00000000000\0",136,"ascii"),s.write("        ",148,"ascii"),s.write("0",156,"ascii"),s.write("ustar\0",257,"ascii"),s.write("00",263,"ascii"),n&&s.write(n,345,155,"utf8");let i=0;for(let o=0;o<O;o++)i+=s[o];return s.write(i.toString(8).padStart(6,"0")+"\0 ",148,"ascii"),s}function He(e){let t=Ne(e),r=[];for(let{rel:n,abs:s}of t){let i=fs.readFileSync(s);r.push(je(n,i.length)),r.push(i);let o=(O-i.length%O)%O;o>0&&r.push(Buffer.alloc(o));}return r.push(Buffer.alloc(O*2)),zlib.gzipSync(Buffer.concat(r),{level:9})}var O,X=Je(()=>{O=512;});var Y="EVOLVE_API_KEY";var ie=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai";var ue=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","upstream_not_watchable","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","package_not_retained","package_corrupt","package_missing","internal_error"];function z(e){return typeof e=="string"&&ue.includes(e)}var L=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,r,n,s){super(n),this.name="EvolveApiError",this.status=t,this.code=r,s?.param!==void 0&&(this.param=s.param),s?.details!==void 0&&(this.details=s.details),s?.retryAfterSec!==void 0&&(this.retryAfterSec=s.retryAfterSec),s?.requestId!==void 0&&(this.requestId=s.requestId);}isKnownCode(){return z(this.code)}};async function ye(e){let t=await e.text().catch(()=>""),r=e.headers?.get?.("x-request-id")??void 0,n=Number(e.headers?.get?.("retry-after")),s=Number.isFinite(n)?n:void 0;try{let i=JSON.parse(t);if(i?.error&&typeof i.error=="object"){let o=typeof i.error.code=="string"?i.error.code:"unknown_error",u=typeof i.error.message=="string"?i.error.message:e.statusText;throw new L(e.status,o,u,{param:typeof i.error.param=="string"?i.error.param:void 0,details:i.error.details&&typeof i.error.details=="object"?i.error.details:void 0,retryAfterSec:typeof i.error.retryAfterSec=="number"?i.error.retryAfterSec:s,requestId:typeof i.error.requestId=="string"?i.error.requestId:r})}}catch(i){if(i instanceof L)throw i}throw new L(e.status,"unknown_error",t||e.statusText,{retryAfterSec:s,requestId:r})}var Z=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},Fe=1e3,qe=3e4,Ge=new Set(["COMPLETED","CANCELLED","FAILED"]),We=new Set(["job.completed","job.cancelled","job.failed"]),Ye=2e3,ze=new Set(["COMPLETED","FAILED"]);function te(e,t){let r=t?.apiKey||process.env[Y];if(!r)throw new Error(`${e}() requires an API key. Set ${Y} or pass { apiKey } in config.`);let n=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||ie).replace(/\/$/,"");return {apiKey:r,baseUrl:n}}async function g(e,t,r){let n=await fetch(`${e.baseUrl}${t}`,{...r,headers:{Authorization:`Bearer ${e.apiKey}`,...r?.headers}});return n.ok||await ye(n),n}function Qe(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let r=e.slice(0,t).trim(),n=e.slice(t+1).trim();if(!r||!n)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:r,version:n}}function re(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function B(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function Xe(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function U(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(re),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function T(e,t){let r=e??{};return {items:Array.isArray(r.items)?r.items.map(t):[],nextCursor:r.nextCursor??null,hasMore:r.hasMore===true}}function S(e,t){let r=new URLSearchParams;e?.limit!==void 0&&r.set("limit",String(e.limit)),e?.cursor&&r.set("cursor",e.cursor);for(let[s,i]of Object.entries(t??{}))i!==void 0&&r.set(s,i);let n=r.toString();return n?`?${n}`:""}function ce(e){return {id:e.id,taskKey:e.taskKey,agent:re(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function Ze(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function N(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...T(t,Ze),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function j(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function H(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function ke(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function et(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:ke(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(re),createdAt:e.createdAt}}function tt(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:ke(e.coverage)}}function rt(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(tt)}}function nt(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function ee(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function de(e,t){return t?.aborted?true:e?.name==="AbortError"}function M(e,t){return new Promise((r,n)=>{let s=setTimeout(()=>{t?.removeEventListener("abort",i),r();},e),i=()=>{clearTimeout(s),n(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",i,{once:true});})}function st(e){let t="";return {push(r){t+=r.replace(/\r\n/g,`
+`);let n;for(;(n=t.indexOf(`
+
+`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let i,o,u=[];for(let a of s.split(`
+`)){if(!a||a.startsWith(":"))continue;let l=a.indexOf(":"),d=l===-1?a:a.slice(0,l),c=l===-1?"":a.slice(l+1);c.startsWith(" ")&&(c=c.slice(1)),d==="id"?i=c:d==="event"?o=c:d==="data"&&u.push(c);}(i!==void 0||o!==void 0||u.length>0)&&e({id:i,event:o,data:u.join(`
+`)});}}}}function J(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,i){return n().then(s,i)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let i=await e({limit:t?.limit,cursor:s});for(let o of i.items)yield o;if(!i.nextCursor)return;s=i.nextCursor;}}}}function V(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function at(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}var Re="x-package-sha256",K=class extends Error{constructor(r,n){super(`downloaded bytes do not match the digest the server stated (expected ${r}, got ${n})`);this.expected=r;this.actual=n;}name="EvolveDigestMismatchError"},F=class extends Error{constructor(r,n){super(`download ended early: the server declared ${r} bytes and ${n} arrived`);this.expectedBytes=r;this.receivedBytes=n;}name="EvolveIncompleteDownloadError"};function ot(e,t){let r=e.headers.get("Content-Length");if(r===null)return;let n=Number(r);if(Number.isFinite(n)&&t!==n)throw new F(n,t)}async function it(e,t){let r=e.headers.get(Re);if(!r)return;let{createHash:n}=await import('crypto'),s=n("sha256").update(t).digest("hex");if(s!==r)throw new K(r,s)}function we(e,t){let n=(e.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);if(!n)return t;let s=path.basename(n[1]);return s===""||s==="."||s===".."||/[/\\]/.test(s)||/[\u0000-\u001f]/.test(s)?t:s}function q(e){let t=te("benchmarks",e);async function r(o){let u=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}`);return H(await u.json())}function n(o){if(!o||typeof o!="object")return null;let u=o;return {ref:u.ref,currentCommit:u.currentCommit,latestCommit:u.latestCommit??null,moved:u.moved===true,behindBy:typeof u.behindBy=="number"?u.behindBy:null,checkedAt:u.checkedAt??null,error:u.error??null}}async function s(o,u){let a=Qe(o),l=S(u,{version:a.version}),c=await(await g(t,`/api/benchmarks/${encodeURIComponent(a.name)}${l}`)).json();return {name:c.name,title:c.title??null,description:c.description??null,activeVersion:c.activeVersion?B(c.activeVersion):null,versions:(c.versions||[]).map(B),selectedVersion:c.selectedVersion?B(c.selectedVersion):null,tasks:T(c.tasks,Xe),upstream:n(c.upstream),createdAt:c.createdAt,updatedAt:c.updatedAt}}async function i(o){let u=await g(t,`/api/benchmarks${S(o)}`);return T(await u.json(),a=>({name:a.name,title:a.title??null,description:a.description??null,activeVersion:a.activeVersion?B(a.activeVersion):null,upstream:n(a.upstream)}))}return {list(o){return J(i,o)},get:s,async getActive(o,u){let a=await s(o,u);if(a.activeVersion===null)throw new Z(o);return {name:a.name,title:a.title,description:a.description,activeVersion:a.activeVersion,version:a.activeVersion.version,tasks:a.tasks??{items:[],nextCursor:null,hasMore:false},versions:a.versions??[],createdAt:a.createdAt,updatedAt:a.updatedAt}},async import(o){let u=o.source;if(u?.directory){let{tarGzipDirectory:a}=await Promise.resolve().then(()=>(X(),Q)),l=a(u.directory),d=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version},{bytes:l,filename:"corpus.tar.gz"})});return H(await d.json())}if(u&&"gitUrl"in u&&u.gitUrl){let a=await g(t,"/api/benchmarks/imports",{method:"POST",body:V({benchmarkName:o.benchmarkName,version:o.version,gitUrl:u.gitUrl,ref:u.ref})});return H(await a.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(o,u){let a=u?.pollIntervalMs??Ye,l=null;for(;;){ee(u?.signal);let d=await r(o);if(d.status!==l&&(l=d.status,u?.onStatus?.(d)),ze.has(d.status))return d;await M(a,u?.signal);}},downloadPackage:(async(o,u)=>{let a=await g(t,`/api/benchmarks/imports/${encodeURIComponent(o)}/package`);if(u?.stream){if(!a.body)throw new Error("Package response has no body");return a.body}if(u?.to){if(!a.body)throw new Error("Package response has no body");let d=u.to;await promises.mkdir(d,{recursive:true});let c=path.join(d,we(a,`import-${o}-corpus.tar.gz`)),{createHash:p,randomBytes:w}=await import('crypto'),v=`${c}.${w(8).toString("hex")}.part`,C=p("sha256"),I=0,y=stream.Readable.fromWeb(a.body);y.on("data",f=>{C.update(f),I+=f.length;});try{await promises$1.pipeline(y,fs.createWriteStream(v));let f=a.headers.get("Content-Length");if(f!==null&&I!==Number(f))throw new F(Number(f),I);let P=a.headers.get(Re),E=C.digest("hex");if(P&&E!==P)throw new K(P,E);await promises.rename(v,c);}catch(f){throw await promises.rm(v,{force:true}).catch(()=>{}),f}return c}let l=Buffer.from(await a.arrayBuffer());return ot(a,l.length),await it(a,l),l}),listImports(o){return J(async u=>{let a=new URLSearchParams;u.limit!==void 0&&a.set("limit",String(u.limit)),u.cursor!==void 0&&a.set("cursor",u.cursor),o?.status!==void 0&&a.set("status",o.status),o?.benchmark!==void 0&&a.set("benchmark",o.benchmark);let l=a.toString()?`?${a}`:"",d=await g(t,`/api/benchmarks/imports${l}`);return T(await d.json(),H)},o)},async delete(o){await g(t,`/api/benchmarks/${encodeURIComponent(o)}`,{method:"DELETE"});}}}function ve(e){let t=te("customHarnesses",e);return {async create(r){let n=await me("customHarnesses().create()",r),s=await g(t,"/api/custom-harnesses",{method:"POST",body:n});return j(await s.json())},list(r){return J(async n=>{let s=await g(t,`/api/custom-harnesses${S(n)}`);return T(await s.json(),j)},r)},async get(r){let n=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return j(await n.json())},async delete(r){await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await me("customHarnesses().upsert()",{...n,name:r}),i=await g(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return j(await i.json())}}}async function me(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:i}=await Promise.resolve().then(()=>(X(),Q)),o=i(t.directory);return V(s,{bytes:o,filename:"source.tar.gz"})}return V(s)}function k(e){let t=te("jobs",e);async function r(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}`);return U(await l.json())}async function n(a){let l=await g(t,`/api/jobs${S(a)}`);return T(await l.json(),U)}async function s(a,l){let d=S(l,{status:l?.status?.length?l.status.join(","):void 0}),c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials${d}`);return T(await c.json(),ce)}async function i(a,l,d){let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/trace${S(d)}`);return T(await c.json(),nt)}async function o(a,l){let d=l?`?format=${encodeURIComponent(l)}`:"";return g(t,`/api/jobs/${encodeURIComponent(a)}/export${d}`)}async function*u(a,l){let{onEvent:d,signal:c}=l??{},p=l?.reconnectDelayMs??Fe,w=l?.maxReconnectDelayMs??qe,v=null,C=false,I=false,y=p;for(;!C;){ee(c);let f;try{f=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(a)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...v!==null?{"Last-Event-ID":String(v)}:{}},signal:c});}catch(R){if(de(R,c))throw R;await M(y,c),y=Math.min(y*2,w);continue}if(!f.ok){if(f.status===429||f.status>=500){await f.text().catch(()=>""),await M(y,c),y=Math.min(y*2,w);continue}await ye(f);}if(!f.body)throw new Error("Event stream response has no body");let P=false,E=[],Ee=st(R=>{let A=Number(R.id),_={seq:Number.isInteger(A)?A:-1,type:R.event||"message",data:R.data?ut(R.data):{}};Number.isInteger(A)&&(v=A),P=true,We.has(_.type)&&(C=true),E.push(_);}),ae=f.body.getReader(),Ae=new TextDecoder;try{for(;!C;){let{done:R,value:A}=await ae.read();if(R)break;Ee.push(Ae.decode(A,{stream:!0}));for(let _ of E)d?.(_),yield _;E.length=0;}}catch(R){if(de(R,c))throw R}finally{await ae.cancel().catch(()=>{});}if(C)break;ee(c);let oe=await r(a);if(Ge.has(oe.status)){if(I)return oe;I=true;continue}P&&(y=p),await M(y,c),y=Math.min(y*2,w);}return r(a)}return {async run(a,l){let d=await g(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:{}},body:JSON.stringify(a)});return U(await d.json())},get:r,list(a){return J(n,a)},trials(a,l){return J(d=>s(a,{...d,status:l?.status}),l)},async trial(a,l){let c=await(await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}`)).json();return {...ce(c),jobId:c.jobId}},trialTrace:i,async*trialTraceEvents(a,l,d){let c=d?.cursor;for(;;){let p=await i(a,l,{cursor:c,limit:d?.limit});for(let w of p.items)yield w;if(!p.nextCursor)return;c=p.nextCursor;}},watch(a,l){return at(u(a,l))},async cancel(a){let l=await g(t,`/api/jobs/${encodeURIComponent(a)}/cancel`,{method:"POST"});return U(await l.json())},async compare(a){let l=a.map(encodeURIComponent).join(","),c=await(await g(t,`/api/jobs/compare?ids=${l}`)).json();return {jobs:(c.jobs||[]).map(et),taskMatrix:(c.taskMatrix||[]).map(rt)}},async rerunFailed(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/rerun-failed`,{method:"POST",headers:l?.idempotencyKey?{"Idempotency-Key":l.idempotencyKey}:void 0});return U(await d.json())},async regrade(a,l){let d={};l?.status?.length&&(d.status=l.status),l?.taskKey!==void 0&&(d.taskKey=l.taskKey);let c=await g(t,`/api/jobs/${encodeURIComponent(a)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return N(await c.json())},async regradeTrial(a,l){let d=await g(t,`/api/jobs/${encodeURIComponent(a)}/trials/${encodeURIComponent(l)}/regrade`,{method:"POST"});return N(await d.json())},listRegrades(a){return J(async l=>{let c=await(await g(t,`/api/regrades${S(l,{jobId:a?.jobId})}`)).json();return {items:c.items.map(N),nextCursor:c.nextCursor,hasMore:c.hasMore}},a)},async getRegrade(a,l){let d=await g(t,`/api/regrades/${encodeURIComponent(a)}${S(l)}`);return N(await d.json())},export:(async(a,l)=>{let d=await o(a,l?.format);if(l?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(l?.to){if(!d.body)throw new Error("Export response has no body");let p=l.to;await promises.mkdir(p,{recursive:true});let w=path.join(p,we(d,`job-${a}-export.json.gz`)),v=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(v,fs.createWriteStream(w)),w}let c=await d.arrayBuffer();return Buffer.from(c)})}}function ut(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var dt=`evolve-evals \u2014 Evolve hosted jobs CLI
+
+Usage: evolve-evals <command> [options]
+
+Commands:
+  run                               Create a job (add --watch to follow it)
+  list                              List your jobs (newest first)
+  get <id>                          Show one job
+  trials <id>                       List a job's trials
+  trial <id> <trial-id>             Show one trial in full detail
+  trace <id> <trial-id>             Print a trial's trace events
+  compare <id> <id> [...]           Compare 2-5 jobs side by side
+  cancel <id>                       Request cancellation of a job
+  rerun-failed <id>                 New job from a terminal job's failed trials
+  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)
+  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)
+  export <id>                       Download the research archive (gzipped JSON)
+  benchmarks                        List the benchmark catalog
+  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)
+  import                            Import a benchmark from a git source or a local directory (--watch to follow)
+  import status <id>                Show one import job
+  download <import-id>              Download the original corpus package (owner only)
+  custom-harnesses                  List your registered custom harnesses
+  custom-harnesses get <name>       Show one custom harness
+  custom-harnesses add              Register a custom harness (install script or local directory)
+  custom-harnesses remove <name>    Delete a custom harness
+  help                              Show this help
+
+Run options:
+  --benchmark <name[@version]>        Benchmark (required; bare name = active version)
+  --tasks <k1,k2,...>                 Task keys (default: every task of the version)
+  --agent <harness:model[:version]>   Agent; repeatable (at least one required)
+  --effort <value>                    Reasoning effort for EVERY agent (values:
+                                      GET /api/meta limits.job.reasoningEfforts).
+                                      Applied verbatim \u2014 an agent whose harness
+                                      cannot honor it is refused by the server,
+                                      never silently skipped. Per-agent efforts
+                                      need the SDK. Omitted: the server default.
+  --runs <n>                          Runs per task x agent (default 1)
+  --concurrency <n>                   Parallel trials (default 1)
+  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)
+  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b
+  --watch                             Stream events until the job finishes
+
+Trial options:
+  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)
+
+Regrade options (whole-job regrade only):
+  --status <s1,s2,...>                Only regrade source trials in these statuses
+  --task <key>                        Only regrade source trials of this task
+
+Trace options:
+  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)
+  --limit <n>                         Max events per page
+
+Import options (a git source OR a local directory; --name and --version required):
+  --git <url>                         Git repository URL (with --ref)
+  --ref <ref>                         Git ref: branch, tag, or commit (with --git)
+  --dir <path>                        Local corpus directory (tarred + uploaded)
+  --name <benchmark>                  Catalog benchmark name to create or extend (required)
+  --version <v>                       Version label for the imported version (required)
+  --watch                             Poll until the import is IMPORTED or FAILED
+
+Custom-harness options ("custom-harnesses add"; an install script OR a local directory):
+  --name <harness>                    Harness name, later used in --agent (required)
+  --install-script <path>             Install script file; its contents are uploaded
+  --dir <path>                        Local harness directory (tarred + uploaded)
+  --run <command>                     Run command, executed with sh -c (required)
+  --env KEY=VALUE                     Env injected at run time; repeatable
+
+Other options:
+  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection
+                                      (list, trials, trace, benchmarks, benchmarks get,
+                                      custom-harnesses, regrade-job)
+  --to <dir>                          Directory to save into, for export and download (default: current dir)
+  --format harbor                     Export the Harbor job-layout bundle
+  --json                              Machine-readable JSON output
+  --api-key <key>                     API key (default: $EVOLVE_API_KEY)
+  --base-url <url>                    API base URL (default: the Evolve dashboard API)`,m=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},mt={json:"boolean","api-key":"string","base-url":"string"},gt={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},Se={command:"help",positionals:[],flags:{}};function pt(e){if(e.length===0)throw new m("No command given");if(e[0]==="--help"||e[0]==="-h")return Se;let t=e[0],r=gt[t];if(!r)throw new m(`Unknown command "${t}"`);let n={},s=[];for(let i=1;i<e.length;i++){let o=e[i];if(o==="--help"||o==="-h")return Se;if(!o.startsWith("--")){s.push(o);continue}let u=o.slice(2),a,l=u.indexOf("=");l!==-1&&(a=u.slice(l+1),u=u.slice(0,l));let d=r.flags[u]??mt[u];if(!d)throw new m(`Unknown option --${u} for "${t}"`);if(d==="boolean"){if(a!==void 0)throw new m(`Option --${u} takes no value`);n[u]=true;continue}let c=a;if(c===void 0){let p=e[i+1];if(p===void 0||p.startsWith("--"))throw new m(`Option --${u} requires a value`);c=p,i++;}if(d==="number"){let p=Number(c);if(c.trim()===""||!Number.isFinite(p))throw new m(`Option --${u} expects a number, got "${c}"`);n[u]=p;}else if(d==="repeat"){let p=n[u]??[];p.push(c),n[u]=p;}else n[u]=c;}if(s.length<r.minPositionals)throw new m(`"${t}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(s.length>r.maxPositionals)throw new m(`"${t}" got unexpected argument "${s[r.maxPositionals]}"`);for(let i of r.required??[])if(!(i in n))throw new m(`"${t}" requires --${i}`);return {command:t,positionals:s,flags:n}}function ft(e){let t=e.indexOf(":");if(t<=0||t===e.length-1)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);let r=e.slice(0,t),n=e.slice(t+1),s=n.indexOf(":");if(s===-1)return {harness:r,model:n};let i=n.slice(0,s),o=n.slice(s+1);if(!i||!o)throw new m(`Invalid --agent "${e}": expected harness:model[:version]`);return {harness:r,model:i,harnessVersion:o}}function bt(e){let t=e.flags,r;if(t.tasks!==void 0&&(r=String(t.tasks).split(",").map(n=>n.trim()).filter(Boolean),r.length===0))throw new m("--tasks got an empty task list");return {benchmark:t.benchmark,...r!==void 0?{tasks:r}:{},agents:t.agent.map(n=>{let s=ft(n);return t.effort!==void 0?{...s,reasoningEffort:t.effort}:s}),...t.runs!==void 0?{runsPerTask:t.runs}:{},...t.concurrency!==void 0?{concurrency:t.concurrency}:{},...t["max-trial-spend"]!==void 0?{maxTrialSpendUsd:t["max-trial-spend"]}:{},...t.provider!==void 0?{sandboxProvider:t.provider}:{}}}function ht(e){let t=e.flags,r=typeof t.dir=="string",n=typeof t.git=="string"||typeof t.ref=="string";if(r&&n)throw new m('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let s of ["name","version"])if(typeof t[s]!="string")throw new m(`"import" requires --${s}`);return {source:{directory:t.dir},benchmarkName:t.name,version:t.version}}for(let s of ["git","ref","name","version"])if(typeof t[s]!="string"){let i=s==="git"||s==="ref"?" (or --dir for a local corpus directory)":"";throw new m(`"import" requires --${s}${i}`)}return {source:{gitUrl:t.git,ref:t.ref},benchmarkName:t.name,version:t.version}}function yt(e){let t={};for(let r of e){let n=r.indexOf("=");if(n<=0)throw new m(`Invalid --env "${r}": expected KEY=VALUE`);t[r.slice(0,n)]=r.slice(n+1);}return t}function kt(e,t=r=>fs.readFileSync(r,"utf-8")){let r=e.flags,n=typeof r.dir=="string",s=typeof r["install-script"]=="string";if(n&&s)throw new m('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!s)throw new m('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let o of ["name","run"])if(typeof r[o]!="string")throw new m(`"custom-harnesses add" requires --${o}`);let i=yt(r.env??[]);return {name:r.name,...n?{directory:r.dir}:{installScript:t(r["install-script"])},runCommand:r.run,...Object.keys(i).length>0?{env:i}:{}}}var Rt={out:e=>process.stdout.write(e+`
+`),err:e=>process.stderr.write(e+`
+`)};function h(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join("  ").trimEnd())}function x(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function se(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function W(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function D(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(se).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",x(e.maxTrialSpendUsd)]),t.push(["worst case",x(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",x(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function wt(e){return [e.id,e.status,e.benchmark,String(e.trials.total),$(e.meanReward),x(e.spentUsd),e.createdAt]}function vt(e){return [e.taskKey,se(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",x(e.spentUsd),e.id]}function St(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",se(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];return e.metrics&&Object.keys(e.metrics).length>0&&t.push(["metrics",Object.entries(e.metrics).map(([r,n])=>`${r}=${n}`).join(" \xB7 ")]),t.push(["spent",x(e.spentUsd)]),e.sandboxProvider&&t.push(["provider",e.sandboxProvider]),e.verifierMode&&t.push(["verifier",e.verifierMode]),e.resolvedHarnessVersion&&t.push(["harness version",e.resolvedHarnessVersion]),e.phaseTimingsMs&&Object.keys(e.phaseTimingsMs).length>0&&t.push(["timings",Object.entries(e.phaseTimingsMs).map(([r,n])=>`${r}=${n}ms`).join(" \xB7 ")]),e.failurePhase&&t.push(["failure phase",e.failurePhase]),e.failureDetail&&t.push(["failure detail",e.failureDetail]),e.sessionRef&&t.push(["session",e.sessionRef]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}function Tt(e){if(e===null)return "-";let t=Math.round(e*1e3)/1e3;return t>0?`+${t}`:String(t)}function Ct(e){return [e.taskKey,e.status,$(e.sourceReward),$(e.reward),Tt(e.rewardDelta),e.sourceTrialId]}function xe(e){let t=[["job id",e.id],["status",e.status],["source job",e.sourceJobId],["provider",e.sandboxProvider],["results",String(e.results.total)]],r=Object.entries(e.results.byStatus).filter(([,s])=>s>0).map(([s,i])=>`${s} ${i}`).join(" \xB7 ");if(r&&t.push(["by status",r]),e.filter&&(e.filter.status?.length||e.filter.taskKey)){let s=[];e.filter.status?.length&&s.push(`status=${e.filter.status.join(",")}`),e.filter.taskKey&&s.push(`task=${e.filter.taskKey}`),t.push(["filter",s.join(" \xB7 ")]);}t.push(["created",e.createdAt]);let n=h(t);if(e.results.items.length>0){n.push("");let s=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let i of e.results.items)s.push(Ct(i));n.push(...h(s)),e.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${e.id} --cursor ${e.results.nextCursor}`);}return n}function Te(e){let t=[["name",e.name],["source",e.source],["run command",e.runCommand]],r=Object.keys(e.env??{});return r.length>0&&t.push(["env",r.sort().join(", ")]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),h(t)}var Ie=["e2b","daytona","modal"];function xt(e){return Ie.filter(t=>e?.[t]!==void 0).map(t=>`${t} ${e[t].ok?"ok":"NO"}`).join(" \xB7 ")}function $(e){return e!==null?String(Math.round(e*1e3)/1e3):"-"}function It(e){let t=W(JSON.stringify(e.data??{}),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${t}`.trimEnd()}function Pe(e){let t=e.failures?.length?` (${e.failures.length} task failure${e.failures.length===1?"":"s"})`:"";return `${e.message}${t}`}function ne(e){let t=[["id",e.id],["status",e.status]];if(e.benchmarkName!==void 0&&t.push(["benchmark",e.benchmarkName]),e.version!==void 0&&t.push(["version",e.version]),e.taskCount!==void 0&&t.push(["tasks",String(e.taskCount)]),e.failure){t.push(["failure",Pe(e.failure)]);for(let r of e.failure.failures??[])t.push([`  ${r.taskKey}`,r.error]);}return h(t)}function Pt(e){let t=[];return e.taskCount!==void 0&&t.push(`tasks=${e.taskCount}`),e.failure&&t.push(W(Pe(e.failure),140)),`status ${e.status.padEnd(12)} ${t.join(" ")}`.trimEnd()}function Et(e){let t={...e.data??{}},r=[];typeof t.trialId=="string"&&r.push(t.trialId),typeof t.taskKey=="string"&&r.push(t.taskKey),typeof t.status=="string"&&r.push(t.status),typeof t.reward=="number"&&r.push(`reward=${t.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[i,o]of Object.entries(t))n.has(i)||r.push(`${i}=${typeof o=="object"?JSON.stringify(o):String(o)}`);let s=W(r.join(" "),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${s}`.trimEnd()}function b(e){let t={};return typeof e.flags["api-key"]=="string"&&(t.apiKey=e.flags["api-key"]),typeof e.flags["base-url"]=="string"&&(t.baseUrl=e.flags["base-url"]),t}function G(e){return {...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{}}}function At(e){return e.status==="COMPLETED"?0:e.status==="FAILED"||e.status==="CANCELLED"?1:0}async function Ot(e,t){let r=bt(e),n=e.flags.json===true,s=e.flags.watch===true,i=k(b(e)),o=await i.run(r);if(!s){if(n)t.out(JSON.stringify(o));else {for(let a of D(o))t.out(a);t.out(""),t.out(`Follow it with: evolve-evals get ${o.id}`);}return 0}n?t.out(JSON.stringify({kind:"job.created",job:o})):t.out(`Job ${o.id} (${o.benchmark}) ${o.status} \u2014 watching\u2026`);let u=await i.watch(o.id,{onEvent:a=>{t.out(n?JSON.stringify({kind:"event",...a}):Et(a));}});if(n)t.out(JSON.stringify({kind:"job.final",job:u}));else {t.out("");for(let a of D(u))t.out(a);}return At(u)}async function Jt(e,t){let n=await k(b(e)).list({...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(n)),0;if(n.items.length===0)return t.out("No jobs."),0;let s=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let i of n.items)s.push(wt(i));for(let i of h(s))t.out(i);return n.nextCursor&&t.out(`
+More: evolve-evals list --cursor ${n.nextCursor}`),0}async function _t(e,t){let n=await k(b(e)).get(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function Ut(e,t){let r=k(b(e)),n;if(e.flags.status!==void 0&&(n=String(e.flags.status).split(",").map(o=>o.trim()).filter(Boolean),n.length===0))throw new m("--status got an empty status list");let s=await r.trials(e.positionals[0],{...n!==void 0?{status:n}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(s)),0;if(s.items.length===0)return t.out("No trials."),0;let i=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let o of s.items)i.push(vt(o));for(let o of h(i))t.out(o);return t.out(`
+${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function Lt(e,t){let n=await k(b(e)).trial(e.positionals[0],e.positionals[1]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of St(n))t.out(s);return 0}async function Dt(e,t){let r=k(b(e)),n=e.flags.json===true,s=0;for await(let i of r.trialTraceEvents(e.positionals[0],e.positionals[1],{...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{}}))t.out(n?JSON.stringify(i):It(i)),s+=1;return !n&&s===0&&t.out("No trace events."),0}function $t(e){let t=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of e.jobs)r.push([n.id,n.benchmark,n.status,$(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,x(n.spentUsd)]);if(t.push(...h(r)),e.taskMatrix.length>0){t.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...e.jobs.map((i,o)=>`JOB ${o+1}`)]],s=e.jobs.map(i=>i.id);for(let i of e.taskMatrix){let o=new Map(i.cells.map(u=>[u.jobId,u]));n.push([i.taskKey,i.disagreement?"!":"",...s.map(u=>{let a=o.get(u);return a?a.meanReward!==null?`${a.status} ${$(a.meanReward)}`:a.status:"-"})]);}t.push(...h(n));}return t}async function Bt(e,t){let n=await k(b(e)).compare(e.positionals);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of $t(n))t.out(s);return 0}async function Nt(e,t){let n=await k(b(e)).cancel(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of D(n))t.out(s);return 0}async function jt(e,t){let n=await k(b(e)).rerunFailed(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else {for(let s of D(n))t.out(s);t.out(""),t.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function Ht(e,t){let r=k(b(e)),[n,s]=e.positionals,i;if(s!==void 0){if(e.flags.status!==void 0||e.flags.task!==void 0)throw new m("--status/--task apply to a whole-job regrade, not a single trial");i=await r.regradeTrial(n,s);}else {let o={};if(e.flags.status!==void 0){let u=String(e.flags.status).split(",").map(a=>a.trim()).filter(Boolean);if(u.length===0)throw new m("--status got an empty status list");o.status=u;}e.flags.task!==void 0&&(o.taskKey=String(e.flags.task)),i=await r.regrade(n,o);}if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of xe(i))t.out(o);t.out(""),t.out(`Follow it with: evolve-evals regrade-job ${i.id}`);}return 0}async function Mt(e,t){let n=await k(b(e)).getRegrade(e.positionals[0],G(e));if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of xe(n))t.out(s);return 0}async function Vt(e,t){let r=e.flags.format;if(r!==void 0&&r!=="harbor")throw new m(`Unknown --format "${r}" (supported: harbor)`);let s=await k(b(e)).export(e.positionals[0],{to:e.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return e.flags.json===true?t.out(JSON.stringify({path:s})):t.out(`Saved ${s}`),0}async function Kt(e,t){let n=await q(b(e)).downloadPackage(e.positionals[0],{to:e.flags.to??process.cwd()});return e.flags.json===true?t.out(JSON.stringify({path:n})):t.out(`Saved ${n}`),0}function Ft(e){let t=h([["name",e.name],["title",e.title??"-"],["description",e.description??"-"],["active version",e.activeVersion?.version??"-"]]);if(e.versions&&e.versions.length>0){t.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let n of e.versions)r.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);t.push(...h(r));}if(e.tasks&&e.tasks.items.length>0){t.push("",`Tasks (version ${e.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let s of e.tasks.items)r.push([s.taskKey,`${s.agentTimeoutSec}s`,`${s.verifierTimeoutSec}s`,xt(s.providers)]);t.push(...h(r)),e.tasks.nextCursor&&t.push(`More tasks: evolve-evals benchmarks get ${e.name} --cursor ${e.tasks.nextCursor}`);let n=new Map;for(let s of e.tasks.items)for(let i of Ie){let o=s.providers?.[i];o&&!o.ok&&!n.has(`${i}:${o.reason}`)&&n.set(`${i}:${o.reason}`,`${i}: ${o.reason}`);}if(n.size>0){t.push("","Provider limitations:");for(let s of n.values())t.push(`  ${s}`);}}return t}async function qt(e,t){let r=q(b(e)),[n,s]=e.positionals;if(n===void 0){let o=await r.list(G(e));if(e.flags.json===true)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No benchmarks."),0;let u=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let a of o.items)u.push([a.name,a.activeVersion?.version??"-",a.activeVersion?.state??"-",a.activeVersion?String(a.activeVersion.taskCount):"-",a.title??"-"]);for(let a of h(u))t.out(a);for(let a of Ce(o.items))t.out(a);return 0}if(n!=="get")throw new m(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!s)throw new m('"benchmarks get" requires a <name[@version]> ref');let i=await r.get(s,G(e));if(e.flags.json===true)t.out(JSON.stringify(i));else {for(let o of Ft(i))t.out(o);for(let o of Ce([i]))t.out(o);}return 0}function Ce(e){let t=[];for(let r of e){if(!r.upstream?.moved)continue;let n=r.activeVersion?`@${r.activeVersion.version}`:"";t.push(`${r.name}${n} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return t}async function Gt(e,t){let[r,n]=e.positionals,s=e.flags.json===true,i=q(b(e));if(r==="status"){if(!n)throw new m('"import status" requires an <id>');let l=await i.getImport(n);if(s)t.out(JSON.stringify(l));else for(let d of ne(l))t.out(d);return 0}if(r!==void 0)throw new m(`Unknown import subcommand "${r}" (supported: status)`);let o=ht(e),u=await i.import(o);if(e.flags.watch!==true){if(s)t.out(JSON.stringify(u));else {for(let l of ne(u))t.out(l);t.out(""),t.out(`Follow it with: evolve-evals import status ${u.id}`);}return 0}s?t.out(JSON.stringify({kind:"import.created",benchmarkImport:u})):t.out(`Import ${u.id} (${o.benchmarkName}) ${u.status} \u2014 watching\u2026`);let a=await i.watchImport(u.id,{onStatus:l=>{t.out(s?JSON.stringify({kind:"import.status",benchmarkImport:l}):Pt(l));}});if(s)t.out(JSON.stringify({kind:"import.final",benchmarkImport:a}));else {t.out("");for(let l of ne(a))t.out(l);}return a.status==="FAILED"?1:0}async function Wt(e,t){let r=ve(b(e)),[n,s]=e.positionals,i=e.flags.json===true;if(n===void 0){let o=await r.list(G(e));if(i)return t.out(JSON.stringify(o)),0;if(o.items.length===0)return t.out("No custom harnesses."),0;let u=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let a of o.items)u.push([a.name,a.source,W(a.runCommand,60),a.updatedAt]);for(let a of h(u))t.out(a);return 0}if(n==="add"){if(s!==void 0)throw new m(`"custom-harnesses add" got unexpected argument "${s}"`);let o=await r.create(kt(e));if(i)t.out(JSON.stringify(o));else {for(let u of Te(o))t.out(u);t.out(""),t.out(`Use it with: evolve-evals run --agent ${o.name}:<model> \u2026`);}return 0}if(n==="get"){if(!s)throw new m('"custom-harnesses get" requires a <name>');let o=await r.get(s);if(i)t.out(JSON.stringify(o));else for(let u of Te(o))t.out(u);return 0}if(n==="remove"){if(!s)throw new m('"custom-harnesses remove" requires a <name>');return await r.delete(s),i?t.out(JSON.stringify({name:s,deleted:true})):t.out(`Deleted custom harness ${s}`),0}throw new m(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function Yt(e,t=Rt){let r;try{r=pt(e);}catch(n){if(n instanceof m)return t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(r.command){case "help":return t.out(dt),0;case "run":return await Ot(r,t);case "list":return await Jt(r,t);case "get":return await _t(r,t);case "trials":return await Ut(r,t);case "trial":return await Lt(r,t);case "trace":return await Dt(r,t);case "compare":return await Bt(r,t);case "cancel":return await Nt(r,t);case "rerun-failed":return await jt(r,t);case "regrade":return await Ht(r,t);case "regrade-job":return await Mt(r,t);case "export":return await Vt(r,t);case "benchmarks":return await qt(r,t);case "import":return await Gt(r,t);case "download":return await Kt(r,t);case "custom-harnesses":return await Wt(r,t);default:return t.err(`Error: unknown command "${r.command}"`),2}}catch(n){return n instanceof m?(t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2):(t.err(`Error: ${n.message}`),1)}}var zt=(()=>{let e=process.argv[1];if(!e)return  false;try{return (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('cli.cjs', document.baseURI).href))===url.pathToFileURL(e).href}catch{return  false}})();zt&&Yt(process.argv.slice(2)).then(e=>{process.exitCode=e;},e=>{process.stderr.write(`Error: ${e?.message??e}
+`),process.exitCode=1;});exports.CliUsageError=m;exports.USAGE=dt;exports.buildCustomHarnessInput=kt;exports.buildImportInput=ht;exports.buildJobInput=bt;exports.eventLine=Et;exports.importStatusLine=Pt;exports.parseArgs=pt;exports.parseHarnessEnv=yt;exports.parseJobAgent=ft;exports.runCli=Yt;exports.traceEventLine=It;

package/dist/hosted/cli.d.cts

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BnWkwyuA.cjs';
+
+/**
+ * evolve-evals — CLI for Evolve hosted benchmarks & jobs.
+ *
+ * Thin shell over the hosted client (benchmarks() / jobs()): plain node
+ * arg parsing, no dependencies. Human-readable tables by default; --json emits
+ * machine-readable JSON (NDJSON for --watch event streams).
+ *
+ * Exit codes: 0 success (watch: job COMPLETED / import IMPORTED), 1
+ * runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
+ */
+
+declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n  run                               Create a job (add --watch to follow it)\n  list                              List your jobs (newest first)\n  get <id>                          Show one job\n  trials <id>                       List a job's trials\n  trial <id> <trial-id>             Show one trial in full detail\n  trace <id> <trial-id>             Print a trial's trace events\n  compare <id> <id> [...]           Compare 2-5 jobs side by side\n  cancel <id>                       Request cancellation of a job\n  rerun-failed <id>                 New job from a terminal job's failed trials\n  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)\n  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)\n  export <id>                       Download the research archive (gzipped JSON)\n  benchmarks                        List the benchmark catalog\n  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)\n  import                            Import a benchmark from a git source or a local directory (--watch to follow)\n  import status <id>                Show one import job\n  download <import-id>              Download the original corpus package (owner only)\n  custom-harnesses                  List your registered custom harnesses\n  custom-harnesses get <name>       Show one custom harness\n  custom-harnesses add              Register a custom harness (install script or local directory)\n  custom-harnesses remove <name>    Delete a custom harness\n  help                              Show this help\n\nRun options:\n  --benchmark <name[@version]>        Benchmark (required; bare name = active version)\n  --tasks <k1,k2,...>                 Task keys (default: every task of the version)\n  --agent <harness:model[:version]>   Agent; repeatable (at least one required)\n  --effort <value>                    Reasoning effort for EVERY agent (values:\n                                      GET /api/meta limits.job.reasoningEfforts).\n                                      Applied verbatim \u2014 an agent whose harness\n                                      cannot honor it is refused by the server,\n                                      never silently skipped. Per-agent efforts\n                                      need the SDK. Omitted: the server default.\n  --runs <n>                          Runs per task x agent (default 1)\n  --concurrency <n>                   Parallel trials (default 1)\n  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)\n  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b\n  --watch                             Stream events until the job finishes\n\nTrial options:\n  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n  --status <s1,s2,...>                Only regrade source trials in these statuses\n  --task <key>                        Only regrade source trials of this task\n\nTrace options:\n  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)\n  --limit <n>                         Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n  --git <url>                         Git repository URL (with --ref)\n  --ref <ref>                         Git ref: branch, tag, or commit (with --git)\n  --dir <path>                        Local corpus directory (tarred + uploaded)\n  --name <benchmark>                  Catalog benchmark name to create or extend (required)\n  --version <v>                       Version label for the imported version (required)\n  --watch                             Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n  --name <harness>                    Harness name, later used in --agent (required)\n  --install-script <path>             Install script file; its contents are uploaded\n  --dir <path>                        Local harness directory (tarred + uploaded)\n  --run <command>                     Run command, executed with sh -c (required)\n  --env KEY=VALUE                     Env injected at run time; repeatable\n\nOther options:\n  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection\n                                      (list, trials, trace, benchmarks, benchmarks get,\n                                      custom-harnesses, regrade-job)\n  --to <dir>                          Directory to save into, for export and download (default: current dir)\n  --format harbor                     Export the Harbor job-layout bundle\n  --json                              Machine-readable JSON output\n  --api-key <key>                     API key (default: $EVOLVE_API_KEY)\n  --base-url <url>                    API base URL (default: the Evolve dashboard API)";
+/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
+declare class CliUsageError extends Error {
+    constructor(message: string);
+}
+interface Invocation {
+    command: string;
+    positionals: string[];
+    flags: Record<string, string | number | boolean | string[]>;
+}
+declare function parseArgs(argv: string[]): Invocation;
+/** Parse "harness:model[:version]" (version may itself contain colons). */
+declare function parseJobAgent(spec: string): JobAgent;
+/**
+ * Build the POST /api/jobs body from a parsed `run` invocation.
+ * Keys follow the contract field order: benchmark, tasks, agents,
+ * runsPerTask, concurrency, maxTrialSpendUsd, sandboxProvider.
+ */
+declare function buildJobInput(inv: Invocation): JobInput;
+/** Build the benchmarks().import() input from a parsed `import` invocation. */
+declare function buildImportInput(inv: Invocation): BenchmarkImportInput;
+/** Parse repeatable `--env KEY=VALUE` into the declared run-time env map. */
+declare function parseHarnessEnv(pairs: string[]): Record<string, string>;
+/**
+ * Build the customHarnesses().create() input from a parsed
+ * `custom-harnesses add` invocation. `--install-script` names a FILE; its
+ * contents are what the SDK uploads.
+ */
+declare function buildCustomHarnessInput(inv: Invocation, readScript?: (path: string) => string): CustomHarnessInput;
+interface CliIO {
+    out(line: string): void;
+    err(line: string): void;
+}
+/** One trace event line — evolve-evals trace. */
+declare function traceEventLine(event: TrialTraceEvent): string;
+/** Compact one-line rendering of one import status change for --watch. */
+declare function importStatusLine(job: BenchmarkImport): string;
+/** Compact one-line rendering of one SSE event for --watch. */
+declare function eventLine(event: JobEvent): string;
+declare function runCli(argv: string[], io?: CliIO): Promise<number>;
+
+export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };

package/dist/hosted/cli.d.ts

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BnWkwyuA.js';
+
+/**
+ * evolve-evals — CLI for Evolve hosted benchmarks & jobs.
+ *
+ * Thin shell over the hosted client (benchmarks() / jobs()): plain node
+ * arg parsing, no dependencies. Human-readable tables by default; --json emits
+ * machine-readable JSON (NDJSON for --watch event streams).
+ *
+ * Exit codes: 0 success (watch: job COMPLETED / import IMPORTED), 1
+ * runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
+ */
+
+declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n  run                               Create a job (add --watch to follow it)\n  list                              List your jobs (newest first)\n  get <id>                          Show one job\n  trials <id>                       List a job's trials\n  trial <id> <trial-id>             Show one trial in full detail\n  trace <id> <trial-id>             Print a trial's trace events\n  compare <id> <id> [...]           Compare 2-5 jobs side by side\n  cancel <id>                       Request cancellation of a job\n  rerun-failed <id>                 New job from a terminal job's failed trials\n  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)\n  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)\n  export <id>                       Download the research archive (gzipped JSON)\n  benchmarks                        List the benchmark catalog\n  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)\n  import                            Import a benchmark from a git source or a local directory (--watch to follow)\n  import status <id>                Show one import job\n  download <import-id>              Download the original corpus package (owner only)\n  custom-harnesses                  List your registered custom harnesses\n  custom-harnesses get <name>       Show one custom harness\n  custom-harnesses add              Register a custom harness (install script or local directory)\n  custom-harnesses remove <name>    Delete a custom harness\n  help                              Show this help\n\nRun options:\n  --benchmark <name[@version]>        Benchmark (required; bare name = active version)\n  --tasks <k1,k2,...>                 Task keys (default: every task of the version)\n  --agent <harness:model[:version]>   Agent; repeatable (at least one required)\n  --effort <value>                    Reasoning effort for EVERY agent (values:\n                                      GET /api/meta limits.job.reasoningEfforts).\n                                      Applied verbatim \u2014 an agent whose harness\n                                      cannot honor it is refused by the server,\n                                      never silently skipped. Per-agent efforts\n                                      need the SDK. Omitted: the server default.\n  --runs <n>                          Runs per task x agent (default 1)\n  --concurrency <n>                   Parallel trials (default 1)\n  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)\n  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b\n  --watch                             Stream events until the job finishes\n\nTrial options:\n  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n  --status <s1,s2,...>                Only regrade source trials in these statuses\n  --task <key>                        Only regrade source trials of this task\n\nTrace options:\n  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)\n  --limit <n>                         Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n  --git <url>                         Git repository URL (with --ref)\n  --ref <ref>                         Git ref: branch, tag, or commit (with --git)\n  --dir <path>                        Local corpus directory (tarred + uploaded)\n  --name <benchmark>                  Catalog benchmark name to create or extend (required)\n  --version <v>                       Version label for the imported version (required)\n  --watch                             Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n  --name <harness>                    Harness name, later used in --agent (required)\n  --install-script <path>             Install script file; its contents are uploaded\n  --dir <path>                        Local harness directory (tarred + uploaded)\n  --run <command>                     Run command, executed with sh -c (required)\n  --env KEY=VALUE                     Env injected at run time; repeatable\n\nOther options:\n  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection\n                                      (list, trials, trace, benchmarks, benchmarks get,\n                                      custom-harnesses, regrade-job)\n  --to <dir>                          Directory to save into, for export and download (default: current dir)\n  --format harbor                     Export the Harbor job-layout bundle\n  --json                              Machine-readable JSON output\n  --api-key <key>                     API key (default: $EVOLVE_API_KEY)\n  --base-url <url>                    API base URL (default: the Evolve dashboard API)";
+/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
+declare class CliUsageError extends Error {
+    constructor(message: string);
+}
+interface Invocation {
+    command: string;
+    positionals: string[];
+    flags: Record<string, string | number | boolean | string[]>;
+}
+declare function parseArgs(argv: string[]): Invocation;
+/** Parse "harness:model[:version]" (version may itself contain colons). */
+declare function parseJobAgent(spec: string): JobAgent;
+/**
+ * Build the POST /api/jobs body from a parsed `run` invocation.
+ * Keys follow the contract field order: benchmark, tasks, agents,
+ * runsPerTask, concurrency, maxTrialSpendUsd, sandboxProvider.
+ */
+declare function buildJobInput(inv: Invocation): JobInput;
+/** Build the benchmarks().import() input from a parsed `import` invocation. */
+declare function buildImportInput(inv: Invocation): BenchmarkImportInput;
+/** Parse repeatable `--env KEY=VALUE` into the declared run-time env map. */
+declare function parseHarnessEnv(pairs: string[]): Record<string, string>;
+/**
+ * Build the customHarnesses().create() input from a parsed
+ * `custom-harnesses add` invocation. `--install-script` names a FILE; its
+ * contents are what the SDK uploads.
+ */
+declare function buildCustomHarnessInput(inv: Invocation, readScript?: (path: string) => string): CustomHarnessInput;
+interface CliIO {
+    out(line: string): void;
+    err(line: string): void;
+}
+/** One trace event line — evolve-evals trace. */
+declare function traceEventLine(event: TrialTraceEvent): string;
+/** Compact one-line rendering of one import status change for --watch. */
+declare function importStatusLine(job: BenchmarkImport): string;
+/** Compact one-line rendering of one SSE event for --watch. */
+declare function eventLine(event: JobEvent): string;
+declare function runCli(argv: string[], io?: CliIO): Promise<number>;
+
+export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };

package/dist/hosted/cli.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+import {I as I$1,H as H$1,J as J$1}from'../chunk-ZYTUS4TY.js';import {readFileSync}from'fs';import {pathToFileURL}from'url';var j=`evolve-evals \u2014 Evolve hosted jobs CLI
+
+Usage: evolve-evals <command> [options]
+
+Commands:
+  run                               Create a job (add --watch to follow it)
+  list                              List your jobs (newest first)
+  get <id>                          Show one job
+  trials <id>                       List a job's trials
+  trial <id> <trial-id>             Show one trial in full detail
+  trace <id> <trial-id>             Print a trial's trace events
+  compare <id> <id> [...]           Compare 2-5 jobs side by side
+  cancel <id>                       Request cancellation of a job
+  rerun-failed <id>                 New job from a terminal job's failed trials
+  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)
+  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)
+  export <id>                       Download the research archive (gzipped JSON)
+  benchmarks                        List the benchmark catalog
+  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)
+  import                            Import a benchmark from a git source or a local directory (--watch to follow)
+  import status <id>                Show one import job
+  download <import-id>              Download the original corpus package (owner only)
+  custom-harnesses                  List your registered custom harnesses
+  custom-harnesses get <name>       Show one custom harness
+  custom-harnesses add              Register a custom harness (install script or local directory)
+  custom-harnesses remove <name>    Delete a custom harness
+  help                              Show this help
+
+Run options:
+  --benchmark <name[@version]>        Benchmark (required; bare name = active version)
+  --tasks <k1,k2,...>                 Task keys (default: every task of the version)
+  --agent <harness:model[:version]>   Agent; repeatable (at least one required)
+  --effort <value>                    Reasoning effort for EVERY agent (values:
+                                      GET /api/meta limits.job.reasoningEfforts).
+                                      Applied verbatim \u2014 an agent whose harness
+                                      cannot honor it is refused by the server,
+                                      never silently skipped. Per-agent efforts
+                                      need the SDK. Omitted: the server default.
+  --runs <n>                          Runs per task x agent (default 1)
+  --concurrency <n>                   Parallel trials (default 1)
+  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)
+  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b
+  --watch                             Stream events until the job finishes
+
+Trial options:
+  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)
+
+Regrade options (whole-job regrade only):
+  --status <s1,s2,...>                Only regrade source trials in these statuses
+  --task <key>                        Only regrade source trials of this task
+
+Trace options:
+  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)
+  --limit <n>                         Max events per page
+
+Import options (a git source OR a local directory; --name and --version required):
+  --git <url>                         Git repository URL (with --ref)
+  --ref <ref>                         Git ref: branch, tag, or commit (with --git)
+  --dir <path>                        Local corpus directory (tarred + uploaded)
+  --name <benchmark>                  Catalog benchmark name to create or extend (required)
+  --version <v>                       Version label for the imported version (required)
+  --watch                             Poll until the import is IMPORTED or FAILED
+
+Custom-harness options ("custom-harnesses add"; an install script OR a local directory):
+  --name <harness>                    Harness name, later used in --agent (required)
+  --install-script <path>             Install script file; its contents are uploaded
+  --dir <path>                        Local harness directory (tarred + uploaded)
+  --run <command>                     Run command, executed with sh -c (required)
+  --env KEY=VALUE                     Env injected at run time; repeatable
+
+Other options:
+  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection
+                                      (list, trials, trace, benchmarks, benchmarks get,
+                                      custom-harnesses, regrade-job)
+  --to <dir>                          Directory to save into, for export and download (default: current dir)
+  --format harbor                     Export the Harbor job-layout bundle
+  --json                              Machine-readable JSON output
+  --api-key <key>                     API key (default: $EVOLVE_API_KEY)
+  --base-url <url>                    API base URL (default: the Evolve dashboard API)`,l=class extends Error{constructor(e){super(e),this.name="CliUsageError";}},J={json:"boolean","api-key":"string","base-url":"string"},U={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},download:{flags:{to:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<import-id>"},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},O={command:"help",positionals:[],flags:{}};function D(t){if(t.length===0)throw new l("No command given");if(t[0]==="--help"||t[0]==="-h")return O;let e=t[0],n=U[e];if(!n)throw new l(`Unknown command "${e}"`);let s={},r=[];for(let o=1;o<t.length;o++){let i=t[o];if(i==="--help"||i==="-h")return O;if(!i.startsWith("--")){r.push(i);continue}let a=i.slice(2),u,d=a.indexOf("=");d!==-1&&(u=a.slice(d+1),a=a.slice(0,d));let w=n.flags[a]??J[a];if(!w)throw new l(`Unknown option --${a} for "${e}"`);if(w==="boolean"){if(u!==void 0)throw new l(`Option --${a} takes no value`);s[a]=true;continue}let p=u;if(p===void 0){let g=t[o+1];if(g===void 0||g.startsWith("--"))throw new l(`Option --${a} requires a value`);p=g,o++;}if(w==="number"){let g=Number(p);if(p.trim()===""||!Number.isFinite(g))throw new l(`Option --${a} expects a number, got "${p}"`);s[a]=g;}else if(w==="repeat"){let g=s[a]??[];g.push(p),s[a]=g;}else s[a]=p;}if(r.length<n.minPositionals)throw new l(`"${e}" requires ${n.positionalUsage??`${n.minPositionals} argument(s)`}`);if(r.length>n.maxPositionals)throw new l(`"${e}" got unexpected argument "${r[n.maxPositionals]}"`);for(let o of n.required??[])if(!(o in s))throw new l(`"${e}" requires --${o}`);return {command:e,positionals:r,flags:s}}function L(t){let e=t.indexOf(":");if(e<=0||e===t.length-1)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);let n=t.slice(0,e),s=t.slice(e+1),r=s.indexOf(":");if(r===-1)return {harness:n,model:s};let o=s.slice(0,r),i=s.slice(r+1);if(!o||!i)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);return {harness:n,model:o,harnessVersion:i}}function q(t){let e=t.flags,n;if(e.tasks!==void 0&&(n=String(e.tasks).split(",").map(s=>s.trim()).filter(Boolean),n.length===0))throw new l("--tasks got an empty task list");return {benchmark:e.benchmark,...n!==void 0?{tasks:n}:{},agents:e.agent.map(s=>{let r=L(s);return e.effort!==void 0?{...r,reasoningEffort:e.effort}:r}),...e.runs!==void 0?{runsPerTask:e.runs}:{},...e.concurrency!==void 0?{concurrency:e.concurrency}:{},...e["max-trial-spend"]!==void 0?{maxTrialSpendUsd:e["max-trial-spend"]}:{},...e.provider!==void 0?{sandboxProvider:e.provider}:{}}}function M(t){let e=t.flags,n=typeof e.dir=="string",s=typeof e.git=="string"||typeof e.ref=="string";if(n&&s)throw new l('"import" takes EITHER --dir OR --git/--ref, not both');if(n){for(let r of ["name","version"])if(typeof e[r]!="string")throw new l(`"import" requires --${r}`);return {source:{directory:e.dir},benchmarkName:e.name,version:e.version}}for(let r of ["git","ref","name","version"])if(typeof e[r]!="string"){let o=r==="git"||r==="ref"?" (or --dir for a local corpus directory)":"";throw new l(`"import" requires --${r}${o}`)}return {source:{gitUrl:e.git,ref:e.ref},benchmarkName:e.name,version:e.version}}function K(t){let e={};for(let n of t){let s=n.indexOf("=");if(s<=0)throw new l(`Invalid --env "${n}": expected KEY=VALUE`);e[n.slice(0,s)]=n.slice(s+1);}return e}function V(t,e=n=>readFileSync(n,"utf-8")){let n=t.flags,s=typeof n.dir=="string",r=typeof n["install-script"]=="string";if(s&&r)throw new l('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!s&&!r)throw new l('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let i of ["name","run"])if(typeof n[i]!="string")throw new l(`"custom-harnesses add" requires --${i}`);let o=K(n.env??[]);return {name:n.name,...s?{directory:n.dir}:{installScript:e(n["install-script"])},runCommand:n.run,...Object.keys(o).length>0?{env:o}:{}}}var F={out:t=>process.stdout.write(t+`
+`),err:t=>process.stderr.write(t+`
+`)};function f(t){if(t.length===0)return [];let e=[];for(let n of t)n.forEach((s,r)=>{e[r]=Math.max(e[r]??0,s.length);});return t.map(n=>n.map((s,r)=>s.padEnd(e[r])).join("  ").trimEnd())}function h(t){return typeof t=="number"?`$${t.toFixed(2)}`:"-"}function $(t){let e=`${t.harness}:${t.model}`;return t.harnessVersion?`${e}:${t.harnessVersion}`:e}function S(t,e){return t.length>e?t.slice(0,e-1)+"\u2026":t}function b(t){let e=[["id",t.id],["status",t.status],["benchmark",t.benchmark]];e.push(["agents",t.agents.map($).join(", ")]),e.push(["size",`${t.counts.agents} agent(s) x ${t.counts.tasks} task(s) = ${t.trials.total} trial(s)`]),e.push(["runs/task",String(t.runsPerTask)]),e.push(["concurrency",String(t.concurrency)]),e.push(["max spend/trial",h(t.maxTrialSpendUsd)]),e.push(["worst case",h(t.worstCaseSpendUsd)]),e.push(["provider",t.sandboxProvider]),e.push(["spent",h(t.spentUsd)]),e.push(["mean reward",t.meanReward!==null?String(t.meanReward):"-"]);let n=Object.entries(t.trials.byStatus).filter(([,s])=>s>0).map(([s,r])=>`${s} ${r}`).join(" \xB7 ");return n&&e.push(["trials",n]),t.sourceJobId&&e.push(["rerun of",t.sourceJobId]),t.idempotentReplay&&e.push(["note","idempotent replay of an existing job"]),t.failure&&e.push(["failure",`${t.failure.code}: ${t.failure.message}`]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function H(t){return [t.id,t.status,t.benchmark,String(t.trials.total),k(t.meanReward),h(t.spentUsd),t.createdAt]}function B(t){return [t.taskKey,$(t.agent),String(t.runNumber),t.status,t.reward!==null?String(t.reward):"-",h(t.spentUsd),t.id]}function G(t){let e=[["trial id",t.id],["job",t.jobId],["task",t.taskKey],["agent",$(t.agent)],["run",String(t.runNumber)],["status",t.status],["reward",t.reward!==null?String(t.reward):"-"]];return t.metrics&&Object.keys(t.metrics).length>0&&e.push(["metrics",Object.entries(t.metrics).map(([n,s])=>`${n}=${s}`).join(" \xB7 ")]),e.push(["spent",h(t.spentUsd)]),t.sandboxProvider&&e.push(["provider",t.sandboxProvider]),t.verifierMode&&e.push(["verifier",t.verifierMode]),t.resolvedHarnessVersion&&e.push(["harness version",t.resolvedHarnessVersion]),t.phaseTimingsMs&&Object.keys(t.phaseTimingsMs).length>0&&e.push(["timings",Object.entries(t.phaseTimingsMs).map(([n,s])=>`${n}=${s}ms`).join(" \xB7 ")]),t.failurePhase&&e.push(["failure phase",t.failurePhase]),t.failureDetail&&e.push(["failure detail",t.failureDetail]),t.sessionRef&&e.push(["session",t.sessionRef]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}function _(t){if(t===null)return "-";let e=Math.round(t*1e3)/1e3;return e>0?`+${e}`:String(e)}function W(t){return [t.taskKey,t.status,k(t.sourceReward),k(t.reward),_(t.rewardDelta),t.sourceTrialId]}function x(t){let e=[["job id",t.id],["status",t.status],["source job",t.sourceJobId],["provider",t.sandboxProvider],["results",String(t.results.total)]],n=Object.entries(t.results.byStatus).filter(([,r])=>r>0).map(([r,o])=>`${r} ${o}`).join(" \xB7 ");if(n&&e.push(["by status",n]),t.filter&&(t.filter.status?.length||t.filter.taskKey)){let r=[];t.filter.status?.length&&r.push(`status=${t.filter.status.join(",")}`),t.filter.taskKey&&r.push(`task=${t.filter.taskKey}`),e.push(["filter",r.join(" \xB7 ")]);}e.push(["created",t.createdAt]);let s=f(e);if(t.results.items.length>0){s.push("");let r=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let o of t.results.items)r.push(W(o));s.push(...f(r)),t.results.nextCursor&&s.push("",`More: evolve-evals regrade-job ${t.id} --cursor ${t.results.nextCursor}`);}return s}function P(t){let e=[["name",t.name],["source",t.source],["run command",t.runCommand]],n=Object.keys(t.env??{});return n.length>0&&e.push(["env",n.sort().join(", ")]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),f(e)}var C=["e2b","daytona","modal"];function Y(t){return C.filter(e=>t?.[e]!==void 0).map(e=>`${e} ${t[e].ok?"ok":"NO"}`).join(" \xB7 ")}function k(t){return t!==null?String(Math.round(t*1e3)/1e3):"-"}function z(t){let e=S(JSON.stringify(t.data??{}),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${e}`.trimEnd()}function T(t){let e=t.failures?.length?` (${t.failures.length} task failure${t.failures.length===1?"":"s"})`:"";return `${t.message}${e}`}function I(t){let e=[["id",t.id],["status",t.status]];if(t.benchmarkName!==void 0&&e.push(["benchmark",t.benchmarkName]),t.version!==void 0&&e.push(["version",t.version]),t.taskCount!==void 0&&e.push(["tasks",String(t.taskCount)]),t.failure){e.push(["failure",T(t.failure)]);for(let n of t.failure.failures??[])e.push([`  ${n.taskKey}`,n.error]);}return f(e)}function Q(t){let e=[];return t.taskCount!==void 0&&e.push(`tasks=${t.taskCount}`),t.failure&&e.push(S(T(t.failure),140)),`status ${t.status.padEnd(12)} ${e.join(" ")}`.trimEnd()}function X(t){let e={...t.data??{}},n=[];typeof e.trialId=="string"&&n.push(e.trialId),typeof e.taskKey=="string"&&n.push(e.taskKey),typeof e.status=="string"&&n.push(e.status),typeof e.reward=="number"&&n.push(`reward=${e.reward}`);let s=new Set(["trialId","taskKey","status","reward"]);for(let[o,i]of Object.entries(e))s.has(o)||n.push(`${o}=${typeof i=="object"?JSON.stringify(i):String(i)}`);let r=S(n.join(" "),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${r}`.trimEnd()}function c(t){let e={};return typeof t.flags["api-key"]=="string"&&(e.apiKey=t.flags["api-key"]),typeof t.flags["base-url"]=="string"&&(e.baseUrl=t.flags["base-url"]),e}function v(t){return {...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{}}}function Z(t){return t.status==="COMPLETED"?0:t.status==="FAILED"||t.status==="CANCELLED"?1:0}async function tt(t,e){let n=q(t),s=t.flags.json===true,r=t.flags.watch===true,o=J$1(c(t)),i=await o.run(n);if(!r){if(s)e.out(JSON.stringify(i));else {for(let u of b(i))e.out(u);e.out(""),e.out(`Follow it with: evolve-evals get ${i.id}`);}return 0}s?e.out(JSON.stringify({kind:"job.created",job:i})):e.out(`Job ${i.id} (${i.benchmark}) ${i.status} \u2014 watching\u2026`);let a=await o.watch(i.id,{onEvent:u=>{e.out(s?JSON.stringify({kind:"event",...u}):X(u));}});if(s)e.out(JSON.stringify({kind:"job.final",job:a}));else {e.out("");for(let u of b(a))e.out(u);}return Z(a)}async function et(t,e){let s=await J$1(c(t)).list({...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(s)),0;if(s.items.length===0)return e.out("No jobs."),0;let r=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let o of s.items)r.push(H(o));for(let o of f(r))e.out(o);return s.nextCursor&&e.out(`
+More: evolve-evals list --cursor ${s.nextCursor}`),0}async function st(t,e){let s=await J$1(c(t)).get(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let r of b(s))e.out(r);return 0}async function nt(t,e){let n=J$1(c(t)),s;if(t.flags.status!==void 0&&(s=String(t.flags.status).split(",").map(i=>i.trim()).filter(Boolean),s.length===0))throw new l("--status got an empty status list");let r=await n.trials(t.positionals[0],{...s!==void 0?{status:s}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(r)),0;if(r.items.length===0)return e.out("No trials."),0;let o=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let i of r.items)o.push(B(i));for(let i of f(o))e.out(i);return e.out(`
+${r.items.length} trial(s) shown`),r.nextCursor&&e.out(`More: evolve-evals trials ${t.positionals[0]} --cursor ${r.nextCursor}`),0}async function rt(t,e){let s=await J$1(c(t)).trial(t.positionals[0],t.positionals[1]);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let r of G(s))e.out(r);return 0}async function it(t,e){let n=J$1(c(t)),s=t.flags.json===true,r=0;for await(let o of n.trialTraceEvents(t.positionals[0],t.positionals[1],{...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{}}))e.out(s?JSON.stringify(o):z(o)),r+=1;return !s&&r===0&&e.out("No trace events."),0}function ot(t){let e=[],n=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let s of t.jobs)n.push([s.id,s.benchmark,s.status,k(s.meanReward),`${s.coverage.scored}/${s.coverage.total}`,h(s.spentUsd)]);if(e.push(...f(n)),t.taskMatrix.length>0){e.push("","Task matrix (disagreements first; columns in the order above):");let s=[["TASK","DIFF",...t.jobs.map((o,i)=>`JOB ${i+1}`)]],r=t.jobs.map(o=>o.id);for(let o of t.taskMatrix){let i=new Map(o.cells.map(a=>[a.jobId,a]));s.push([o.taskKey,o.disagreement?"!":"",...r.map(a=>{let u=i.get(a);return u?u.meanReward!==null?`${u.status} ${k(u.meanReward)}`:u.status:"-"})]);}e.push(...f(s));}return e}async function at(t,e){let s=await J$1(c(t)).compare(t.positionals);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let r of ot(s))e.out(r);return 0}async function ut(t,e){let s=await J$1(c(t)).cancel(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let r of b(s))e.out(r);return 0}async function lt(t,e){let s=await J$1(c(t)).rerunFailed(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(s));else {for(let r of b(s))e.out(r);e.out(""),e.out(`Follow it with: evolve-evals get ${s.id}`);}return 0}async function ct(t,e){let n=J$1(c(t)),[s,r]=t.positionals,o;if(r!==void 0){if(t.flags.status!==void 0||t.flags.task!==void 0)throw new l("--status/--task apply to a whole-job regrade, not a single trial");o=await n.regradeTrial(s,r);}else {let i={};if(t.flags.status!==void 0){let a=String(t.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(a.length===0)throw new l("--status got an empty status list");i.status=a;}t.flags.task!==void 0&&(i.taskKey=String(t.flags.task)),o=await n.regrade(s,i);}if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of x(o))e.out(i);e.out(""),e.out(`Follow it with: evolve-evals regrade-job ${o.id}`);}return 0}async function ft(t,e){let s=await J$1(c(t)).getRegrade(t.positionals[0],v(t));if(t.flags.json===true)e.out(JSON.stringify(s));else for(let r of x(s))e.out(r);return 0}async function dt(t,e){let n=t.flags.format;if(n!==void 0&&n!=="harbor")throw new l(`Unknown --format "${n}" (supported: harbor)`);let r=await J$1(c(t)).export(t.positionals[0],{to:t.flags.to??process.cwd(),...n==="harbor"?{format:"harbor"}:{}});return t.flags.json===true?e.out(JSON.stringify({path:r})):e.out(`Saved ${r}`),0}async function mt(t,e){let s=await H$1(c(t)).downloadPackage(t.positionals[0],{to:t.flags.to??process.cwd()});return t.flags.json===true?e.out(JSON.stringify({path:s})):e.out(`Saved ${s}`),0}function gt(t){let e=f([["name",t.name],["title",t.title??"-"],["description",t.description??"-"],["active version",t.activeVersion?.version??"-"]]);if(t.versions&&t.versions.length>0){e.push("");let n=[["VERSION","STATE","TASKS","CREATED"]];for(let s of t.versions)n.push([s.version,s.state,String(s.taskCount),s.createdAt??"-"]);e.push(...f(n));}if(t.tasks&&t.tasks.items.length>0){e.push("",`Tasks (version ${t.selectedVersion?.version??"?"}):`);let n=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let r of t.tasks.items)n.push([r.taskKey,`${r.agentTimeoutSec}s`,`${r.verifierTimeoutSec}s`,Y(r.providers)]);e.push(...f(n)),t.tasks.nextCursor&&e.push(`More tasks: evolve-evals benchmarks get ${t.name} --cursor ${t.tasks.nextCursor}`);let s=new Map;for(let r of t.tasks.items)for(let o of C){let i=r.providers?.[o];i&&!i.ok&&!s.has(`${o}:${i.reason}`)&&s.set(`${o}:${i.reason}`,`${o}: ${i.reason}`);}if(s.size>0){e.push("","Provider limitations:");for(let r of s.values())e.push(`  ${r}`);}}return e}async function pt(t,e){let n=H$1(c(t)),[s,r]=t.positionals;if(s===void 0){let i=await n.list(v(t));if(t.flags.json===true)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No benchmarks."),0;let a=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let u of i.items)a.push([u.name,u.activeVersion?.version??"-",u.activeVersion?.state??"-",u.activeVersion?String(u.activeVersion.taskCount):"-",u.title??"-"]);for(let u of f(a))e.out(u);for(let u of R(i.items))e.out(u);return 0}if(s!=="get")throw new l(`Unknown benchmarks subcommand "${s}" (did you mean "benchmarks get ${s}"?)`);if(!r)throw new l('"benchmarks get" requires a <name[@version]> ref');let o=await n.get(r,v(t));if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of gt(o))e.out(i);for(let i of R([o]))e.out(i);}return 0}function R(t){let e=[];for(let n of t){if(!n.upstream?.moved)continue;let s=n.activeVersion?`@${n.activeVersion.version}`:"";e.push(`${n.name}${s} \xB7 upstream ${n.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${n.name} --version <new-version> --git-url <url> --ref ${n.upstream.ref}`);}return e}async function ht(t,e){let[n,s]=t.positionals,r=t.flags.json===true,o=H$1(c(t));if(n==="status"){if(!s)throw new l('"import status" requires an <id>');let d=await o.getImport(s);if(r)e.out(JSON.stringify(d));else for(let w of I(d))e.out(w);return 0}if(n!==void 0)throw new l(`Unknown import subcommand "${n}" (supported: status)`);let i=M(t),a=await o.import(i);if(t.flags.watch!==true){if(r)e.out(JSON.stringify(a));else {for(let d of I(a))e.out(d);e.out(""),e.out(`Follow it with: evolve-evals import status ${a.id}`);}return 0}r?e.out(JSON.stringify({kind:"import.created",benchmarkImport:a})):e.out(`Import ${a.id} (${i.benchmarkName}) ${a.status} \u2014 watching\u2026`);let u=await o.watchImport(a.id,{onStatus:d=>{e.out(r?JSON.stringify({kind:"import.status",benchmarkImport:d}):Q(d));}});if(r)e.out(JSON.stringify({kind:"import.final",benchmarkImport:u}));else {e.out("");for(let d of I(u))e.out(d);}return u.status==="FAILED"?1:0}async function wt(t,e){let n=I$1(c(t)),[s,r]=t.positionals,o=t.flags.json===true;if(s===void 0){let i=await n.list(v(t));if(o)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No custom harnesses."),0;let a=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let u of i.items)a.push([u.name,u.source,S(u.runCommand,60),u.updatedAt]);for(let u of f(a))e.out(u);return 0}if(s==="add"){if(r!==void 0)throw new l(`"custom-harnesses add" got unexpected argument "${r}"`);let i=await n.create(V(t));if(o)e.out(JSON.stringify(i));else {for(let a of P(i))e.out(a);e.out(""),e.out(`Use it with: evolve-evals run --agent ${i.name}:<model> \u2026`);}return 0}if(s==="get"){if(!r)throw new l('"custom-harnesses get" requires a <name>');let i=await n.get(r);if(o)e.out(JSON.stringify(i));else for(let a of P(i))e.out(a);return 0}if(s==="remove"){if(!r)throw new l('"custom-harnesses remove" requires a <name>');return await n.delete(r),o?e.out(JSON.stringify({name:r,deleted:true})):e.out(`Deleted custom harness ${r}`),0}throw new l(`Unknown custom-harnesses subcommand "${s}" (supported: add, get, remove)`)}async function bt(t,e=F){let n;try{n=D(t);}catch(s){if(s instanceof l)return e.err(`Error: ${s.message}`),e.err('Run "evolve-evals help" for usage.'),2;throw s}try{switch(n.command){case "help":return e.out(j),0;case "run":return await tt(n,e);case "list":return await et(n,e);case "get":return await st(n,e);case "trials":return await nt(n,e);case "trial":return await rt(n,e);case "trace":return await it(n,e);case "compare":return await at(n,e);case "cancel":return await ut(n,e);case "rerun-failed":return await lt(n,e);case "regrade":return await ct(n,e);case "regrade-job":return await ft(n,e);case "export":return await dt(n,e);case "benchmarks":return await pt(n,e);case "import":return await ht(n,e);case "download":return await mt(n,e);case "custom-harnesses":return await wt(n,e);default:return e.err(`Error: unknown command "${n.command}"`),2}}catch(s){return s instanceof l?(e.err(`Error: ${s.message}`),e.err('Run "evolve-evals help" for usage.'),2):(e.err(`Error: ${s.message}`),1)}}var kt=(()=>{let t=process.argv[1];if(!t)return  false;try{return import.meta.url===pathToFileURL(t).href}catch{return  false}})();kt&&bt(process.argv.slice(2)).then(t=>{process.exitCode=t;},t=>{process.stderr.write(`Error: ${t?.message??t}
+`),process.exitCode=1;});export{l as CliUsageError,j as USAGE,V as buildCustomHarnessInput,M as buildImportInput,q as buildJobInput,X as eventLine,Q as importStatusLine,D as parseArgs,K as parseHarnessEnv,L as parseJobAgent,bt as runCli,z as traceEventLine};