@evolvingmachines/sdk 0.0.51 → 0.0.52-project-sable.20260726.6e95f36

package/dist/chunk-IHBQMTA4.js

@@ -0,0 +1,6 @@
+import {createWriteStream}from'fs';import {mkdir}from'fs/promises';import {join}from'path';import {Readable}from'stream';import {pipeline}from'stream/promises';function re(e=""){let t=process.env.EVOLVE_GATEWAY_URL||"https://swarmkit-gateway-692833842999.us-central1.run.app";if(!e)return t;let r=e.startsWith("/")?e:`/${e}`;return `${t}${r}`}function ne(e=""){let t=process.env.EVOLVE_DASHBOARD_URL||P;if(!e)return t;let r=e.startsWith("/")?e:`/${e}`;return `${t.replace(/\/$/,"")}${r}`}var Ae=["e2b","daytona","modal"];function q(e){return ne(`/api/managed/${e}`)}function _e(){return `${q("daytona")}/toolbox`}function Oe(){return q("e2b")}function we(e){return {"browser-use":{type:"http",url:re("/browser_use/mcp"),headers:{"x-litellm-api-key":`Bearer ${e}`}}}}var D="EVOLVE_API_KEY",Je="E2B_API_KEY",Le="DAYTONA_API_KEY",Be="MODAL_TOKEN_ID",De="MODAL_TOKEN_SECRET",Ue="claude",He=36e5,Me=4,Ne=100,W="/home/user";function je(e,t){return t?t.replace(/\/+$/,"")||"/":e?e==="root"?"/root":`/home/${e}`:W}var P=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai",$e="x-litellm-customer-id",Ke="x-litellm-tags",Fe="run:",Ge=".evolve-sdk/observability/sessions",qe=100,We=5e3,Ye=3,ze=1e3;var Y=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","internal_error"];function U(e){return typeof e=="string"&&Y.includes(e)}var E=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,r,a,i){super(a),this.name="EvolveApiError",this.status=t,this.code=r,i?.param!==void 0&&(this.param=i.param),i?.details!==void 0&&(this.details=i.details),i?.retryAfterSec!==void 0&&(this.retryAfterSec=i.retryAfterSec),i?.requestId!==void 0&&(this.requestId=i.requestId);}isKnownCode(){return U(this.code)}};async function N(e){let t=await e.text().catch(()=>""),r=e.headers?.get?.("x-request-id")??void 0,a=Number(e.headers?.get?.("retry-after")),i=Number.isFinite(a)?a:void 0;try{let u=JSON.parse(t);if(u?.error&&typeof u.error=="object"){let l=typeof u.error.code=="string"?u.error.code:"unknown_error",c=typeof u.error.message=="string"?u.error.message:e.statusText;throw new E(e.status,l,c,{param:typeof u.error.param=="string"?u.error.param:void 0,details:u.error.details&&typeof u.error.details=="object"?u.error.details:void 0,retryAfterSec:typeof u.error.retryAfterSec=="number"?u.error.retryAfterSec:i,requestId:typeof u.error.requestId=="string"?u.error.requestId:r})}}catch(u){if(u instanceof E)throw u}throw new E(e.status,"unknown_error",t||e.statusText,{retryAfterSec:i,requestId:r})}var H=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},le=1e3,ce=3e4,de=new Set(["COMPLETED","CANCELLED","FAILED"]),me=new Set(["job.completed","job.cancelled","job.failed"]),pe=2e3,ge=new Set(["COMPLETED","FAILED"]);function j(e,t){let r=t?.apiKey||process.env[D];if(!r)throw new Error(`${e}() requires an API key. Set ${D} or pass { apiKey } in config.`);let a=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||P).replace(/\/$/,"");return {apiKey:r,baseUrl:a}}async function p(e,t,r){let a=await fetch(`${e.baseUrl}${t}`,{...r,headers:{Authorization:`Bearer ${e.apiKey}`,...r?.headers}});return a.ok||await N(a),a}function be(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let r=e.slice(0,t).trim(),a=e.slice(t+1).trim();if(!r||!a)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:r,version:a}}function V(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function A(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function fe(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function S(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(V),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function k(e,t){let r=e??{};return {items:Array.isArray(r.items)?r.items.map(t):[],nextCursor:r.nextCursor??null,hasMore:r.hasMore===true}}function y(e,t){let r=new URLSearchParams;e?.limit!==void 0&&r.set("limit",String(e.limit)),e?.cursor&&r.set("cursor",e.cursor);for(let[i,u]of Object.entries(t??{}))u!==void 0&&r.set(i,u);let a=r.toString();return a?`?${a}`:""}function z(e){return {id:e.id,taskKey:e.taskKey,agent:V(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function Re(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function _(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...k(t,Re),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function O(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function w(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function Z(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function he(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:Z(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(V),createdAt:e.createdAt}}function ye(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:Z(e.coverage)}}function ke(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(ye)}}function Te(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function M(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function Q(e,t){return t?.aborted?true:e?.name==="AbortError"}function J(e,t){return new Promise((r,a)=>{let i=setTimeout(()=>{t?.removeEventListener("abort",u),r();},e),u=()=>{clearTimeout(i),a(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",u,{once:true});})}function ve(e){let t="";return {push(r){t+=r.replace(/\r\n/g,`
+`);let a;for(;(a=t.indexOf(`
+
+`))!==-1;){let i=t.slice(0,a);t=t.slice(a+2);let u,l,c=[];for(let d of i.split(`
+`)){if(!d||d.startsWith(":"))continue;let s=d.indexOf(":"),n=s===-1?d:d.slice(0,s),o=s===-1?"":d.slice(s+1);o.startsWith(" ")&&(o=o.slice(1)),n==="id"?u=o:n==="event"?l=o:n==="data"&&c.push(o);}(u!==void 0||l!==void 0||c.length>0)&&e({id:u,event:l,data:c.join(`
+`)});}}}}function v(e,t){let r,a=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(i,u){return a().then(i,u)},catch(i){return a().catch(i)},finally(i){return a().finally(i)},async*[Symbol.asyncIterator](){let i=t?.cursor;for(;;){let u=await e({limit:t?.limit,cursor:i});for(let l of u.items)yield l;if(!u.nextCursor)return;i=u.nextCursor;}}}}function L(e,t){let r=new FormData;for(let[a,i]of Object.entries(e))i!==void 0&&r.set(a,i);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function Ce(e){let t,r=()=>(t||(t=(async()=>{let a=await e.next();for(;!a.done;)a=await e.next();return a.value})()),t);return {then(a,i){return r().then(a,i)},catch(a){return r().catch(a)},finally(a){return r().finally(a)},[Symbol.asyncIterator](){return e}}}function xe(e){let t=j("benchmarks",e);async function r(l){let c=await p(t,`/api/benchmarks/imports/${encodeURIComponent(l)}`);return w(await c.json())}function a(l){if(!l||typeof l!="object")return null;let c=l;return {ref:c.ref,currentCommit:c.currentCommit,latestCommit:c.latestCommit??null,moved:c.moved===true,behindBy:typeof c.behindBy=="number"?c.behindBy:null,checkedAt:c.checkedAt??null,error:c.error??null}}async function i(l,c){let d=be(l),s=y(c,{version:d.version}),o=await(await p(t,`/api/benchmarks/${encodeURIComponent(d.name)}${s}`)).json();return {name:o.name,title:o.title??null,description:o.description??null,activeVersion:o.activeVersion?A(o.activeVersion):null,versions:(o.versions||[]).map(A),selectedVersion:o.selectedVersion?A(o.selectedVersion):null,tasks:k(o.tasks,fe),upstream:a(o.upstream),createdAt:o.createdAt,updatedAt:o.updatedAt}}async function u(l){let c=await p(t,`/api/benchmarks${y(l)}`);return k(await c.json(),d=>({name:d.name,title:d.title??null,description:d.description??null,activeVersion:d.activeVersion?A(d.activeVersion):null,upstream:a(d.upstream)}))}return {list(l){return v(u,l)},get:i,async getActive(l,c){let d=await i(l,c);if(d.activeVersion===null)throw new H(l);return {name:d.name,title:d.title,description:d.description,activeVersion:d.activeVersion,version:d.activeVersion.version,tasks:d.tasks??{items:[],nextCursor:null,hasMore:false},versions:d.versions??[],createdAt:d.createdAt,updatedAt:d.updatedAt}},async import(l){let c=l.source;if(c?.directory){let{tarGzipDirectory:d}=await import('./tar-WPIXS3E6.js'),s=d(c.directory),n=await p(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:l.benchmarkName,version:l.version},{bytes:s,filename:"corpus.tar.gz"})});return w(await n.json())}if(c&&"gitUrl"in c&&c.gitUrl){let d=await p(t,"/api/benchmarks/imports",{method:"POST",body:L({benchmarkName:l.benchmarkName,version:l.version,gitUrl:c.gitUrl,ref:c.ref})});return w(await d.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(l,c){let d=c?.pollIntervalMs??pe,s=null;for(;;){M(c?.signal);let n=await r(l);if(n.status!==s&&(s=n.status,c?.onStatus?.(n)),ge.has(n.status))return n;await J(d,c?.signal);}},listImports(l){return v(async c=>{let d=new URLSearchParams;c.limit!==void 0&&d.set("limit",String(c.limit)),c.cursor!==void 0&&d.set("cursor",c.cursor),l?.status!==void 0&&d.set("status",l.status),l?.benchmark!==void 0&&d.set("benchmark",l.benchmark);let s=d.toString()?`?${d}`:"",n=await p(t,`/api/benchmarks/imports${s}`);return k(await n.json(),w)},l)},async delete(l){await p(t,`/api/benchmarks/${encodeURIComponent(l)}`,{method:"DELETE"});}}}function Se(e){let t=j("customHarnesses",e);return {async create(r){let a=await X("customHarnesses().create()",r),i=await p(t,"/api/custom-harnesses",{method:"POST",body:a});return O(await i.json())},list(r){return v(async a=>{let i=await p(t,`/api/custom-harnesses${y(a)}`);return k(await i.json(),O)},r)},async get(r){let a=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return O(await a.json())},async delete(r){await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,a){let i=await X("customHarnesses().upsert()",{...a,name:r}),u=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:i});return O(await u.json())}}}async function X(e,t){let r=typeof t.installScript=="string",a=typeof t.directory=="string";if(r&&a)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!a)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let i={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(a){let{tarGzipDirectory:u}=await import('./tar-WPIXS3E6.js'),l=u(t.directory);return L(i,{bytes:l,filename:"source.tar.gz"})}return L(i)}function Ee(e){let t=j("jobs",e);async function r(s){let n=await p(t,`/api/jobs/${encodeURIComponent(s)}`);return S(await n.json())}async function a(s){let n=await p(t,`/api/jobs${y(s)}`);return k(await n.json(),S)}async function i(s,n){let o=y(n,{status:n?.status?.length?n.status.join(","):void 0}),m=await p(t,`/api/jobs/${encodeURIComponent(s)}/trials${o}`);return k(await m.json(),z)}async function u(s,n,o){let m=await p(t,`/api/jobs/${encodeURIComponent(s)}/trials/${encodeURIComponent(n)}/trace${y(o)}`);return k(await m.json(),Te)}async function l(s,n){let o=n?`?format=${encodeURIComponent(n)}`:"";return p(t,`/api/jobs/${encodeURIComponent(s)}/export${o}`)}function c(s,n){let m=(s.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);return m?m[1]:`job-${n}-export.json.gz`}async function*d(s,n){let{onEvent:o,signal:m}=n??{},f=n?.reconnectDelayMs??le,R=n?.maxReconnectDelayMs??ce,C=null,I=false,$=false,b=f;for(;!I;){M(m);let h;try{h=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(s)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...C!==null?{"Last-Event-ID":String(C)}:{}},signal:m});}catch(g){if(Q(g,m))throw g;await J(b,m),b=Math.min(b*2,R);continue}if(!h.ok){if(h.status===429||h.status>=500){await h.text().catch(()=>""),await J(b,m),b=Math.min(b*2,R);continue}await N(h);}if(!h.body)throw new Error("Event stream response has no body");let K=false,B=[],ee=ve(g=>{let T=Number(g.id),x={seq:Number.isInteger(T)?T:-1,type:g.event||"message",data:g.data?Ie(g.data):{}};Number.isInteger(T)&&(C=T),K=true,me.has(x.type)&&(I=true),B.push(x);}),F=h.body.getReader(),te=new TextDecoder;try{for(;!I;){let{done:g,value:T}=await F.read();if(g)break;ee.push(te.decode(T,{stream:!0}));for(let x of B)o?.(x),yield x;B.length=0;}}catch(g){if(Q(g,m))throw g}finally{await F.cancel().catch(()=>{});}if(I)break;M(m);let G=await r(s);if(de.has(G.status)){if($)return G;$=true;continue}K&&(b=f),await J(b,m),b=Math.min(b*2,R);}return r(s)}return {async run(s,n){let o=await p(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...n?.idempotencyKey?{"Idempotency-Key":n.idempotencyKey}:{}},body:JSON.stringify(s)});return S(await o.json())},get:r,list(s){return v(a,s)},trials(s,n){return v(o=>i(s,{...o,status:n?.status}),n)},async trial(s,n){let m=await(await p(t,`/api/jobs/${encodeURIComponent(s)}/trials/${encodeURIComponent(n)}`)).json();return {...z(m),jobId:m.jobId}},trialTrace:u,async*trialTraceEvents(s,n,o){let m=o?.cursor;for(;;){let f=await u(s,n,{cursor:m,limit:o?.limit});for(let R of f.items)yield R;if(!f.nextCursor)return;m=f.nextCursor;}},watch(s,n){return Ce(d(s,n))},async cancel(s){let n=await p(t,`/api/jobs/${encodeURIComponent(s)}/cancel`,{method:"POST"});return S(await n.json())},async compare(s){let n=s.map(encodeURIComponent).join(","),m=await(await p(t,`/api/jobs/compare?ids=${n}`)).json();return {jobs:(m.jobs||[]).map(he),taskMatrix:(m.taskMatrix||[]).map(ke)}},async rerunFailed(s,n){let o=await p(t,`/api/jobs/${encodeURIComponent(s)}/rerun-failed`,{method:"POST",headers:n?.idempotencyKey?{"Idempotency-Key":n.idempotencyKey}:void 0});return S(await o.json())},async regrade(s,n){let o={};n?.status?.length&&(o.status=n.status),n?.taskKey!==void 0&&(o.taskKey=n.taskKey);let m=await p(t,`/api/jobs/${encodeURIComponent(s)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(o)});return _(await m.json())},async regradeTrial(s,n){let o=await p(t,`/api/jobs/${encodeURIComponent(s)}/trials/${encodeURIComponent(n)}/regrade`,{method:"POST"});return _(await o.json())},listRegrades(s){return v(async n=>{let m=await(await p(t,`/api/regrades${y(n,{jobId:s?.jobId})}`)).json();return {items:m.items.map(_),nextCursor:m.nextCursor,hasMore:m.hasMore}},s)},async getRegrade(s,n){let o=await p(t,`/api/regrades/${encodeURIComponent(s)}${y(n)}`);return _(await o.json())},export:(async(s,n)=>{let o=await l(s,n?.format);if(n?.stream){if(!o.body)throw new Error("Export response has no body");return o.body}if(n?.to){if(!o.body)throw new Error("Export response has no body");let f=n.to;await mkdir(f,{recursive:true});let R=join(f,c(o,s)),C=Readable.fromWeb(o.body);return await pipeline(C,createWriteStream(R)),R}let m=await o.arrayBuffer();return Buffer.from(m)})}}function Ie(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}function ot(e){let t,r,a;return {get benchmarks(){return t??=xe(e)},get customHarnesses(){return r??=Se(e)},get jobs(){return a??=Ee(e)},meta(){return Pe(e)}}}async function Pe(e){let t=(e?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||P).replace(/\/$/,""),r=await fetch(`${t}/api/meta`);return r.ok||await N(r),await r.json()}export{ze as A,Y as B,U as C,E as D,H as E,xe as F,Se as G,Ee as H,ot as I,Pe as J,re as a,ne as b,Ae as c,q as d,_e as e,Oe as f,we as g,D as h,Je as i,Le as j,Be as k,De as l,Ue as m,He as n,Me as o,Ne as p,W as q,je as r,P as s,$e as t,Ke as u,Fe as v,Ge as w,qe as x,We as y,Ye as z};

package/dist/hosted/cli.cjs

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+'use strict';var zlib=require('zlib'),fs=require('fs'),path=require('path'),url=require('url'),promises=require('fs/promises'),stream=require('stream'),promises$1=require('stream/promises');var _documentCurrentScript=typeof document!=='undefined'?document.currentScript:null;var we=Object.defineProperty;var Te=(e,t)=>()=>(e&&(t=e(e=0)),t);var Ce=(e,t)=>{for(var r in t)we(e,r,{get:t[r],enumerable:true});};var G={};Ce(G,{tarGzipDirectory:()=>_e});function Oe(e){let t=[],r=n=>{let s=n===""?e:path.join(e,n),o=fs.readdirSync(s,{withFileTypes:true}).sort((a,i)=>a.name<i.name?-1:a.name>i.name?1:0);for(let a of o){if(a.name.startsWith("."))continue;let i=n===""?a.name:`${n}/${a.name}`,u=path.join(s,a.name),l=fs.lstatSync(u);l.isSymbolicLink()||(l.isDirectory()?r(i):l.isFile()&&t.push({rel:i,abs:u}));}};return r(""),t.sort((n,s)=>n.rel<s.rel?-1:n.rel>s.rel?1:0)}function Je(e,t){let r=e,n="";if(Buffer.byteLength(r,"utf8")>100){let a=-1;for(let i=r.indexOf("/");i!==-1;i=r.indexOf("/",i+1)){let u=r.slice(0,i),l=r.slice(i+1);l.length>0&&Buffer.byteLength(l,"utf8")<=100&&Buffer.byteLength(u,"utf8")<=155&&(a=i);}if(a===-1)throw new Error(`path too long to store in a USTAR tar (max 255 bytes, split at "/"): ${e}`);n=r.slice(0,a),r=r.slice(a+1);}let s=Buffer.alloc(x);s.write(r,0,100,"utf8"),s.write("0000644\0",100,"ascii"),s.write("0000000\0",108,"ascii"),s.write("0000000\0",116,"ascii"),s.write(t.toString(8).padStart(11,"0")+"\0",124,"ascii"),s.write("00000000000\0",136,"ascii"),s.write("        ",148,"ascii"),s.write("0",156,"ascii"),s.write("ustar\0",257,"ascii"),s.write("00",263,"ascii"),n&&s.write(n,345,155,"utf8");let o=0;for(let a=0;a<x;a++)o+=s[a];return s.write(o.toString(8).padStart(6,"0")+"\0 ",148,"ascii"),s}function _e(e){let t=Oe(e),r=[];for(let{rel:n,abs:s}of t){let o=fs.readFileSync(s);r.push(Je(n,o.length)),r.push(o);let a=(x-o.length%x)%x;a>0&&r.push(Buffer.alloc(a));}return r.push(Buffer.alloc(x*2)),zlib.gzipSync(Buffer.concat(r),{level:9})}var x,W=Te(()=>{x=512;});var F="EVOLVE_API_KEY";var oe=process.env.EVOLVE_DASHBOARD_URL||"https://dashboard.evolvingmachines.ai";var ie=["missing_authorization","invalid_api_key","credential_service_unavailable","rate_limited","insufficient_credits","invalid_json","invalid_input","invalid_limit","invalid_status","invalid_cursor","invalid_after","invalid_format","invalid_ids","invalid_multipart","idempotency_key_reused","benchmark_not_found","benchmark_version_not_found","benchmark_name_taken","benchmark_in_use","benchmark_not_owned","no_active_version","version_not_ready","unknown_task_keys","no_tasks","custom_harness_not_found","custom_harness_name_taken","custom_harness_name_reserved","custom_harness_invalid_name","custom_harness_source_required","custom_harness_source_conflict","custom_harness_invalid_env","custom_harness_too_large","custom_harness_limit_reached","harness_version_not_found","job_too_large","provider_unsupported","job_not_found","job_not_terminal","no_failed_runs","trial_not_found","concurrent_update","regrade_source_ineligible","no_regradable_runs","regrade_not_found","import_not_found","import_too_large","invalid_archive","internal_error"];function q(e){return typeof e=="string"&&ie.includes(e)}var J=class extends Error{status;code;param;details;retryAfterSec;requestId;constructor(t,r,n,s){super(n),this.name="EvolveApiError",this.status=t,this.code=r,s?.param!==void 0&&(this.param=s.param),s?.details!==void 0&&(this.details=s.details),s?.retryAfterSec!==void 0&&(this.retryAfterSec=s.retryAfterSec),s?.requestId!==void 0&&(this.requestId=s.requestId);}isKnownCode(){return q(this.code)}};async function me(e){let t=await e.text().catch(()=>""),r=e.headers?.get?.("x-request-id")??void 0,n=Number(e.headers?.get?.("retry-after")),s=Number.isFinite(n)?n:void 0;try{let o=JSON.parse(t);if(o?.error&&typeof o.error=="object"){let a=typeof o.error.code=="string"?o.error.code:"unknown_error",i=typeof o.error.message=="string"?o.error.message:e.statusText;throw new J(e.status,a,i,{param:typeof o.error.param=="string"?o.error.param:void 0,details:o.error.details&&typeof o.error.details=="object"?o.error.details:void 0,retryAfterSec:typeof o.error.retryAfterSec=="number"?o.error.retryAfterSec:s,requestId:typeof o.error.requestId=="string"?o.error.requestId:r})}}catch(o){if(o instanceof J)throw o}throw new J(e.status,"unknown_error",t||e.statusText,{retryAfterSec:s,requestId:r})}var Y=class extends Error{benchmark;constructor(t){super(`Benchmark "${t}" has no active version`),this.name="NoActiveVersionError",this.benchmark=t;}},Ne=1e3,je=3e4,He=new Set(["COMPLETED","CANCELLED","FAILED"]),Me=new Set(["job.completed","job.cancelled","job.failed"]),Ve=2e3,Ke=new Set(["COMPLETED","FAILED"]);function Q(e,t){let r=t?.apiKey||process.env[F];if(!r)throw new Error(`${e}() requires an API key. Set ${F} or pass { apiKey } in config.`);let n=(t?.baseUrl||process.env.EVOLVE_DASHBOARD_URL||oe).replace(/\/$/,"");return {apiKey:r,baseUrl:n}}async function p(e,t,r){let n=await fetch(`${e.baseUrl}${t}`,{...r,headers:{Authorization:`Bearer ${e.apiKey}`,...r?.headers}});return n.ok||await me(n),n}function Fe(e){let t=e.indexOf("@");if(t===-1)return {name:e.trim()};let r=e.slice(0,t).trim(),n=e.slice(t+1).trim();if(!r||!n)throw new Error(`Invalid benchmark ref "${e}": expected "name" or "name@version"`);return {name:r,version:n}}function X(e){return {harness:e.harness,model:e.model,harnessVersion:e.harnessVersion??null,reasoningEffort:e.reasoningEffort??null}}function D(e){return {version:e.version,state:e.state,createdAt:e.createdAt,taskCount:e.taskCount??0}}function qe(e){return {taskKey:e.taskKey,agentTimeoutSec:e.agentTimeoutSec,verifierTimeoutSec:e.verifierTimeoutSec,providers:e.providers}}function O(e){let t=e.trials??{};return {id:e.id,status:e.status,benchmark:e.benchmark,agents:(e.agents??[]).map(X),runsPerTask:e.runsPerTask,concurrency:e.concurrency,maxTrialSpendUsd:e.maxTrialSpendUsd,worstCaseSpendUsd:e.worstCaseSpendUsd,sandboxProvider:e.sandboxProvider,spentUsd:e.spentUsd??0,createdAt:e.createdAt,updatedAt:e.updatedAt,counts:e.counts,trials:{total:t.total??0,byStatus:t.byStatus??{}},meanReward:e.meanReward??null,failure:e.failure??null,sourceJobId:e.sourceJobId??null,idempotentReplay:e.idempotentReplay===true}}function T(e,t){let r=e??{};return {items:Array.isArray(r.items)?r.items.map(t):[],nextCursor:r.nextCursor??null,hasMore:r.hasMore===true}}function w(e,t){let r=new URLSearchParams;e?.limit!==void 0&&r.set("limit",String(e.limit)),e?.cursor&&r.set("cursor",e.cursor);for(let[s,o]of Object.entries(t??{}))o!==void 0&&r.set(s,o);let n=r.toString();return n?`?${n}`:""}function le(e){return {id:e.id,taskKey:e.taskKey,agent:X(e.agent||{}),runNumber:e.runNumber,status:e.status,reward:e.reward??null,metrics:e.metrics??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,modelUsage:e.modelUsage??null,sandboxProvider:e.sandboxProvider??null,verifierMode:e.verifierMode??null,spentUsd:e.spentUsd??null,spendSource:e.spendSource??null,liveSpentUsd:e.liveSpentUsd??null,liveSpendAt:e.liveSpendAt??null,resolvedHarnessVersion:e.resolvedHarnessVersion??null,sessionRef:e.sessionRef??null,createdAt:e.createdAt,updatedAt:e.updatedAt}}function Ge(e){return {id:e.id,sourceTrialId:e.sourceTrialId,taskKey:e.taskKey,status:e.status,reward:e.reward??null,metrics:e.metrics??null,sourceReward:e.sourceReward??null,sourceStatus:e.sourceStatus,rewardDelta:e.rewardDelta??null,verifierMode:e.verifierMode??"separate",verifierDigest:e.verifierDigest??null,verifierSandboxId:e.verifierSandboxId??null,failurePhase:e.failurePhase??null,failureDetail:e.failureDetail??null,phaseTimingsMs:e.phaseTimingsMs??null,createdAt:e.createdAt,settledAt:e.settledAt??null}}function $(e){let t=e.results??{};return {id:e.id,sourceJobId:e.sourceJobId,status:e.status,sandboxProvider:e.sandboxProvider,filter:e.filter??null,results:{...T(t,Ge),total:t.total??0,byStatus:t.byStatus??{}},createdAt:e.createdAt,updatedAt:e.updatedAt}}function B(e){return {name:e.name,source:e.source,runCommand:e.runCommand,env:e.env??{},createdAt:e.createdAt,updatedAt:e.updatedAt}}function N(e){let t={id:e.id,status:e.status,benchmarkName:e.benchmarkName,version:e.version,failure:e.failure??null};return typeof e.taskCount=="number"&&(t.taskCount=e.taskCount),typeof e.createdAt=="string"&&(t.createdAt=e.createdAt),typeof e.updatedAt=="string"&&(t.updatedAt=e.updatedAt),t}function ge(e){let t=e??{};return {scored:t.scored??0,total:t.total??0}}function We(e){return {id:e.id,benchmark:e.benchmark,status:e.status,meanReward:e.meanReward??null,coverage:ge(e.coverage),spentUsd:e.spentUsd??0,agents:(e.agents||[]).map(X),createdAt:e.createdAt}}function Ye(e){return {jobId:e.jobId,status:e.status,meanReward:e.meanReward??null,coverage:ge(e.coverage)}}function ze(e){return {taskKey:e.taskKey,disagreement:e.disagreement===true,cells:(e.cells||[]).map(Ye)}}function Qe(e){return {seq:e.seq,type:e.type,data:e.data??{}}}function z(e){if(e?.aborted)throw e.reason instanceof Error?e.reason:new Error("Watch aborted")}function ce(e,t){return t?.aborted?true:e?.name==="AbortError"}function j(e,t){return new Promise((r,n)=>{let s=setTimeout(()=>{t?.removeEventListener("abort",o),r();},e),o=()=>{clearTimeout(s),n(t?.reason instanceof Error?t.reason:new Error("Watch aborted"));};t?.addEventListener("abort",o,{once:true});})}function Xe(e){let t="";return {push(r){t+=r.replace(/\r\n/g,`
+`);let n;for(;(n=t.indexOf(`
+
+`))!==-1;){let s=t.slice(0,n);t=t.slice(n+2);let o,a,i=[];for(let u of s.split(`
+`)){if(!u||u.startsWith(":"))continue;let l=u.indexOf(":"),c=l===-1?u:u.slice(0,l),d=l===-1?"":u.slice(l+1);d.startsWith(" ")&&(d=d.slice(1)),c==="id"?o=d:c==="event"?a=d:c==="data"&&i.push(d);}(o!==void 0||a!==void 0||i.length>0)&&e({id:o,event:a,data:i.join(`
+`)});}}}}function E(e,t){let r,n=()=>r??=e({limit:t?.limit,cursor:t?.cursor});return {then(s,o){return n().then(s,o)},catch(s){return n().catch(s)},finally(s){return n().finally(s)},async*[Symbol.asyncIterator](){let s=t?.cursor;for(;;){let o=await e({limit:t?.limit,cursor:s});for(let a of o.items)yield a;if(!o.nextCursor)return;s=o.nextCursor;}}}}function H(e,t){let r=new FormData;for(let[n,s]of Object.entries(e))s!==void 0&&r.set(n,s);return t&&r.set("file",new Blob([t.bytes],{type:"application/gzip"}),t.filename),r}function Ze(e){let t,r=()=>(t||(t=(async()=>{let n=await e.next();for(;!n.done;)n=await e.next();return n.value})()),t);return {then(n,s){return r().then(n,s)},catch(n){return r().catch(n)},finally(n){return r().finally(n)},[Symbol.asyncIterator](){return e}}}function Z(e){let t=Q("benchmarks",e);async function r(a){let i=await p(t,`/api/benchmarks/imports/${encodeURIComponent(a)}`);return N(await i.json())}function n(a){if(!a||typeof a!="object")return null;let i=a;return {ref:i.ref,currentCommit:i.currentCommit,latestCommit:i.latestCommit??null,moved:i.moved===true,behindBy:typeof i.behindBy=="number"?i.behindBy:null,checkedAt:i.checkedAt??null,error:i.error??null}}async function s(a,i){let u=Fe(a),l=w(i,{version:u.version}),d=await(await p(t,`/api/benchmarks/${encodeURIComponent(u.name)}${l}`)).json();return {name:d.name,title:d.title??null,description:d.description??null,activeVersion:d.activeVersion?D(d.activeVersion):null,versions:(d.versions||[]).map(D),selectedVersion:d.selectedVersion?D(d.selectedVersion):null,tasks:T(d.tasks,qe),upstream:n(d.upstream),createdAt:d.createdAt,updatedAt:d.updatedAt}}async function o(a){let i=await p(t,`/api/benchmarks${w(a)}`);return T(await i.json(),u=>({name:u.name,title:u.title??null,description:u.description??null,activeVersion:u.activeVersion?D(u.activeVersion):null,upstream:n(u.upstream)}))}return {list(a){return E(o,a)},get:s,async getActive(a,i){let u=await s(a,i);if(u.activeVersion===null)throw new Y(a);return {name:u.name,title:u.title,description:u.description,activeVersion:u.activeVersion,version:u.activeVersion.version,tasks:u.tasks??{items:[],nextCursor:null,hasMore:false},versions:u.versions??[],createdAt:u.createdAt,updatedAt:u.updatedAt}},async import(a){let i=a.source;if(i?.directory){let{tarGzipDirectory:u}=await Promise.resolve().then(()=>(W(),G)),l=u(i.directory),c=await p(t,"/api/benchmarks/imports",{method:"POST",body:H({benchmarkName:a.benchmarkName,version:a.version},{bytes:l,filename:"corpus.tar.gz"})});return N(await c.json())}if(i&&"gitUrl"in i&&i.gitUrl){let u=await p(t,"/api/benchmarks/imports",{method:"POST",body:H({benchmarkName:a.benchmarkName,version:a.version,gitUrl:i.gitUrl,ref:i.ref})});return N(await u.json())}throw new Error("benchmarks().import() requires either a git source ({ source: { gitUrl, ref } }) or a local corpus directory ({ source: { directory } }), plus benchmarkName and version")},getImport:r,async watchImport(a,i){let u=i?.pollIntervalMs??Ve,l=null;for(;;){z(i?.signal);let c=await r(a);if(c.status!==l&&(l=c.status,i?.onStatus?.(c)),Ke.has(c.status))return c;await j(u,i?.signal);}},listImports(a){return E(async i=>{let u=new URLSearchParams;i.limit!==void 0&&u.set("limit",String(i.limit)),i.cursor!==void 0&&u.set("cursor",i.cursor),a?.status!==void 0&&u.set("status",a.status),a?.benchmark!==void 0&&u.set("benchmark",a.benchmark);let l=u.toString()?`?${u}`:"",c=await p(t,`/api/benchmarks/imports${l}`);return T(await c.json(),N)},a)},async delete(a){await p(t,`/api/benchmarks/${encodeURIComponent(a)}`,{method:"DELETE"});}}}function pe(e){let t=Q("customHarnesses",e);return {async create(r){let n=await de("customHarnesses().create()",r),s=await p(t,"/api/custom-harnesses",{method:"POST",body:n});return B(await s.json())},list(r){return E(async n=>{let s=await p(t,`/api/custom-harnesses${w(n)}`);return T(await s.json(),B)},r)},async get(r){let n=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`);return B(await n.json())},async delete(r){await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"DELETE"});},async upsert(r,n){let s=await de("customHarnesses().upsert()",{...n,name:r}),o=await p(t,`/api/custom-harnesses/${encodeURIComponent(r)}`,{method:"PUT",body:s});return B(await o.json())}}}async function de(e,t){let r=typeof t.installScript=="string",n=typeof t.directory=="string";if(r&&n)throw new Error(`${e} takes EITHER an install script ({ installScript }) or a local directory ({ directory }), not both`);if(!r&&!n)throw new Error(`${e} requires either an install script ({ installScript }) or a local directory ({ directory }), plus runCommand`);let s={name:t.name,runCommand:t.runCommand,...t.env!==void 0?{env:JSON.stringify(t.env)}:{},...r?{installScript:t.installScript}:{}};if(n){let{tarGzipDirectory:o}=await Promise.resolve().then(()=>(W(),G)),a=o(t.directory);return H(s,{bytes:a,filename:"source.tar.gz"})}return H(s)}function h(e){let t=Q("jobs",e);async function r(l){let c=await p(t,`/api/jobs/${encodeURIComponent(l)}`);return O(await c.json())}async function n(l){let c=await p(t,`/api/jobs${w(l)}`);return T(await c.json(),O)}async function s(l,c){let d=w(c,{status:c?.status?.length?c.status.join(","):void 0}),m=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials${d}`);return T(await m.json(),le)}async function o(l,c,d){let m=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/trace${w(d)}`);return T(await m.json(),Qe)}async function a(l,c){let d=c?`?format=${encodeURIComponent(c)}`:"";return p(t,`/api/jobs/${encodeURIComponent(l)}/export${d}`)}function i(l,c){let m=(l.headers.get("Content-Disposition")||"").match(/filename="([^"]+)"/);return m?m[1]:`job-${c}-export.json.gz`}async function*u(l,c){let{onEvent:d,signal:m}=c??{},R=c?.reconnectDelayMs??Ne,v=c?.maxReconnectDelayMs??je,P=null,L=false,re=false,k=R;for(;!L;){z(m);let S;try{S=await fetch(`${t.baseUrl}/api/jobs/${encodeURIComponent(l)}/events`,{headers:{Authorization:`Bearer ${t.apiKey}`,Accept:"text/event-stream",...P!==null?{"Last-Event-ID":String(P)}:{}},signal:m});}catch(y){if(ce(y,m))throw y;await j(k,m),k=Math.min(k*2,v);continue}if(!S.ok){if(S.status===429||S.status>=500){await S.text().catch(()=>""),await j(k,m),k=Math.min(k*2,v);continue}await me(S);}if(!S.body)throw new Error("Event stream response has no body");let ne=false,K=[],ve=Xe(y=>{let I=Number(y.id),A={seq:Number.isInteger(I)?I:-1,type:y.event||"message",data:y.data?et(y.data):{}};Number.isInteger(I)&&(P=I),ne=true,Me.has(A.type)&&(L=true),K.push(A);}),se=S.body.getReader(),Se=new TextDecoder;try{for(;!L;){let{done:y,value:I}=await se.read();if(y)break;ve.push(Se.decode(I,{stream:!0}));for(let A of K)d?.(A),yield A;K.length=0;}}catch(y){if(ce(y,m))throw y}finally{await se.cancel().catch(()=>{});}if(L)break;z(m);let ae=await r(l);if(He.has(ae.status)){if(re)return ae;re=true;continue}ne&&(k=R),await j(k,m),k=Math.min(k*2,v);}return r(l)}return {async run(l,c){let d=await p(t,"/api/jobs",{method:"POST",headers:{"Content-Type":"application/json",...c?.idempotencyKey?{"Idempotency-Key":c.idempotencyKey}:{}},body:JSON.stringify(l)});return O(await d.json())},get:r,list(l){return E(n,l)},trials(l,c){return E(d=>s(l,{...d,status:c?.status}),c)},async trial(l,c){let m=await(await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}`)).json();return {...le(m),jobId:m.jobId}},trialTrace:o,async*trialTraceEvents(l,c,d){let m=d?.cursor;for(;;){let R=await o(l,c,{cursor:m,limit:d?.limit});for(let v of R.items)yield v;if(!R.nextCursor)return;m=R.nextCursor;}},watch(l,c){return Ze(u(l,c))},async cancel(l){let c=await p(t,`/api/jobs/${encodeURIComponent(l)}/cancel`,{method:"POST"});return O(await c.json())},async compare(l){let c=l.map(encodeURIComponent).join(","),m=await(await p(t,`/api/jobs/compare?ids=${c}`)).json();return {jobs:(m.jobs||[]).map(We),taskMatrix:(m.taskMatrix||[]).map(ze)}},async rerunFailed(l,c){let d=await p(t,`/api/jobs/${encodeURIComponent(l)}/rerun-failed`,{method:"POST",headers:c?.idempotencyKey?{"Idempotency-Key":c.idempotencyKey}:void 0});return O(await d.json())},async regrade(l,c){let d={};c?.status?.length&&(d.status=c.status),c?.taskKey!==void 0&&(d.taskKey=c.taskKey);let m=await p(t,`/api/jobs/${encodeURIComponent(l)}/regrade`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d)});return $(await m.json())},async regradeTrial(l,c){let d=await p(t,`/api/jobs/${encodeURIComponent(l)}/trials/${encodeURIComponent(c)}/regrade`,{method:"POST"});return $(await d.json())},listRegrades(l){return E(async c=>{let m=await(await p(t,`/api/regrades${w(c,{jobId:l?.jobId})}`)).json();return {items:m.items.map($),nextCursor:m.nextCursor,hasMore:m.hasMore}},l)},async getRegrade(l,c){let d=await p(t,`/api/regrades/${encodeURIComponent(l)}${w(c)}`);return $(await d.json())},export:(async(l,c)=>{let d=await a(l,c?.format);if(c?.stream){if(!d.body)throw new Error("Export response has no body");return d.body}if(c?.to){if(!d.body)throw new Error("Export response has no body");let R=c.to;await promises.mkdir(R,{recursive:true});let v=path.join(R,i(d,l)),P=stream.Readable.fromWeb(d.body);return await promises$1.pipeline(P,fs.createWriteStream(v)),v}let m=await d.arrayBuffer();return Buffer.from(m)})}}function et(e){try{let t=JSON.parse(e);return t&&typeof t=="object"?t:{value:t}}catch{return {raw:e}}}var nt=`evolve-evals \u2014 Evolve hosted jobs CLI
+
+Usage: evolve-evals <command> [options]
+
+Commands:
+  run                               Create a job (add --watch to follow it)
+  list                              List your jobs (newest first)
+  get <id>                          Show one job
+  trials <id>                       List a job's trials
+  trial <id> <trial-id>             Show one trial in full detail
+  trace <id> <trial-id>             Print a trial's trace events
+  compare <id> <id> [...]           Compare 2-5 jobs side by side
+  cancel <id>                       Request cancellation of a job
+  rerun-failed <id>                 New job from a terminal job's failed trials
+  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)
+  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)
+  export <id>                       Download the research archive (gzipped JSON)
+  benchmarks                        List the benchmark catalog
+  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)
+  import                            Import a benchmark from a git source or a local directory (--watch to follow)
+  import status <id>                Show one import job
+  custom-harnesses                  List your registered custom harnesses
+  custom-harnesses get <name>       Show one custom harness
+  custom-harnesses add              Register a custom harness (install script or local directory)
+  custom-harnesses remove <name>    Delete a custom harness
+  help                              Show this help
+
+Run options:
+  --benchmark <name[@version]>        Benchmark (required; bare name = active version)
+  --tasks <k1,k2,...>                 Task keys (default: every task of the version)
+  --agent <harness:model[:version]>   Agent; repeatable (at least one required)
+  --effort <value>                    Reasoning effort for EVERY agent (values:
+                                      GET /api/meta limits.job.reasoningEfforts).
+                                      Applied verbatim \u2014 an agent whose harness
+                                      cannot honor it is refused by the server,
+                                      never silently skipped. Per-agent efforts
+                                      need the SDK. Omitted: the server default.
+  --runs <n>                          Runs per task x agent (default 1)
+  --concurrency <n>                   Parallel trials (default 1)
+  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)
+  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b
+  --watch                             Stream events until the job finishes
+
+Trial options:
+  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)
+
+Regrade options (whole-job regrade only):
+  --status <s1,s2,...>                Only regrade source trials in these statuses
+  --task <key>                        Only regrade source trials of this task
+
+Trace options:
+  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)
+  --limit <n>                         Max events per page
+
+Import options (a git source OR a local directory; --name and --version required):
+  --git <url>                         Git repository URL (with --ref)
+  --ref <ref>                         Git ref: branch, tag, or commit (with --git)
+  --dir <path>                        Local corpus directory (tarred + uploaded)
+  --name <benchmark>                  Catalog benchmark name to create or extend (required)
+  --version <v>                       Version label for the imported version (required)
+  --watch                             Poll until the import is IMPORTED or FAILED
+
+Custom-harness options ("custom-harnesses add"; an install script OR a local directory):
+  --name <harness>                    Harness name, later used in --agent (required)
+  --install-script <path>             Install script file; its contents are uploaded
+  --dir <path>                        Local harness directory (tarred + uploaded)
+  --run <command>                     Run command, executed with sh -c (required)
+  --env KEY=VALUE                     Env injected at run time; repeatable
+
+Other options:
+  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection
+                                      (list, trials, trace, benchmarks, benchmarks get,
+                                      custom-harnesses, regrade-job)
+  --to <dir>                          Export target directory (default: current dir)
+  --format harbor                     Export the Harbor job-layout bundle
+  --json                              Machine-readable JSON output
+  --api-key <key>                     API key (default: $EVOLVE_API_KEY)
+  --base-url <url>                    API base URL (default: the Evolve dashboard API)`,g=class extends Error{constructor(t){super(t),this.name="CliUsageError";}},st={json:"boolean","api-key":"string","base-url":"string"},at={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},fe={command:"help",positionals:[],flags:{}};function ot(e){if(e.length===0)throw new g("No command given");if(e[0]==="--help"||e[0]==="-h")return fe;let t=e[0],r=at[t];if(!r)throw new g(`Unknown command "${t}"`);let n={},s=[];for(let o=1;o<e.length;o++){let a=e[o];if(a==="--help"||a==="-h")return fe;if(!a.startsWith("--")){s.push(a);continue}let i=a.slice(2),u,l=i.indexOf("=");l!==-1&&(u=i.slice(l+1),i=i.slice(0,l));let c=r.flags[i]??st[i];if(!c)throw new g(`Unknown option --${i} for "${t}"`);if(c==="boolean"){if(u!==void 0)throw new g(`Option --${i} takes no value`);n[i]=true;continue}let d=u;if(d===void 0){let m=e[o+1];if(m===void 0||m.startsWith("--"))throw new g(`Option --${i} requires a value`);d=m,o++;}if(c==="number"){let m=Number(d);if(d.trim()===""||!Number.isFinite(m))throw new g(`Option --${i} expects a number, got "${d}"`);n[i]=m;}else if(c==="repeat"){let m=n[i]??[];m.push(d),n[i]=m;}else n[i]=d;}if(s.length<r.minPositionals)throw new g(`"${t}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(s.length>r.maxPositionals)throw new g(`"${t}" got unexpected argument "${s[r.maxPositionals]}"`);for(let o of r.required??[])if(!(o in n))throw new g(`"${t}" requires --${o}`);return {command:t,positionals:s,flags:n}}function it(e){let t=e.indexOf(":");if(t<=0||t===e.length-1)throw new g(`Invalid --agent "${e}": expected harness:model[:version]`);let r=e.slice(0,t),n=e.slice(t+1),s=n.indexOf(":");if(s===-1)return {harness:r,model:n};let o=n.slice(0,s),a=n.slice(s+1);if(!o||!a)throw new g(`Invalid --agent "${e}": expected harness:model[:version]`);return {harness:r,model:o,harnessVersion:a}}function ut(e){let t=e.flags,r;if(t.tasks!==void 0&&(r=String(t.tasks).split(",").map(n=>n.trim()).filter(Boolean),r.length===0))throw new g("--tasks got an empty task list");return {benchmark:t.benchmark,...r!==void 0?{tasks:r}:{},agents:t.agent.map(n=>{let s=it(n);return t.effort!==void 0?{...s,reasoningEffort:t.effort}:s}),...t.runs!==void 0?{runsPerTask:t.runs}:{},...t.concurrency!==void 0?{concurrency:t.concurrency}:{},...t["max-trial-spend"]!==void 0?{maxTrialSpendUsd:t["max-trial-spend"]}:{},...t.provider!==void 0?{sandboxProvider:t.provider}:{}}}function lt(e){let t=e.flags,r=typeof t.dir=="string",n=typeof t.git=="string"||typeof t.ref=="string";if(r&&n)throw new g('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let s of ["name","version"])if(typeof t[s]!="string")throw new g(`"import" requires --${s}`);return {source:{directory:t.dir},benchmarkName:t.name,version:t.version}}for(let s of ["git","ref","name","version"])if(typeof t[s]!="string"){let o=s==="git"||s==="ref"?" (or --dir for a local corpus directory)":"";throw new g(`"import" requires --${s}${o}`)}return {source:{gitUrl:t.git,ref:t.ref},benchmarkName:t.name,version:t.version}}function ct(e){let t={};for(let r of e){let n=r.indexOf("=");if(n<=0)throw new g(`Invalid --env "${r}": expected KEY=VALUE`);t[r.slice(0,n)]=r.slice(n+1);}return t}function dt(e,t=r=>fs.readFileSync(r,"utf-8")){let r=e.flags,n=typeof r.dir=="string",s=typeof r["install-script"]=="string";if(n&&s)throw new g('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!n&&!s)throw new g('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let a of ["name","run"])if(typeof r[a]!="string")throw new g(`"custom-harnesses add" requires --${a}`);let o=ct(r.env??[]);return {name:r.name,...n?{directory:r.dir}:{installScript:t(r["install-script"])},runCommand:r.run,...Object.keys(o).length>0?{env:o}:{}}}var mt={out:e=>process.stdout.write(e+`
+`),err:e=>process.stderr.write(e+`
+`)};function f(e){if(e.length===0)return [];let t=[];for(let r of e)r.forEach((n,s)=>{t[s]=Math.max(t[s]??0,n.length);});return e.map(r=>r.map((n,s)=>n.padEnd(t[s])).join("  ").trimEnd())}function C(e){return typeof e=="number"?`$${e.toFixed(2)}`:"-"}function te(e){let t=`${e.harness}:${e.model}`;return e.harnessVersion?`${t}:${e.harnessVersion}`:t}function V(e,t){return e.length>t?e.slice(0,t-1)+"\u2026":e}function _(e){let t=[["id",e.id],["status",e.status],["benchmark",e.benchmark]];t.push(["agents",e.agents.map(te).join(", ")]),t.push(["size",`${e.counts.agents} agent(s) x ${e.counts.tasks} task(s) = ${e.trials.total} trial(s)`]),t.push(["runs/task",String(e.runsPerTask)]),t.push(["concurrency",String(e.concurrency)]),t.push(["max spend/trial",C(e.maxTrialSpendUsd)]),t.push(["worst case",C(e.worstCaseSpendUsd)]),t.push(["provider",e.sandboxProvider]),t.push(["spent",C(e.spentUsd)]),t.push(["mean reward",e.meanReward!==null?String(e.meanReward):"-"]);let r=Object.entries(e.trials.byStatus).filter(([,n])=>n>0).map(([n,s])=>`${n} ${s}`).join(" \xB7 ");return r&&t.push(["trials",r]),e.sourceJobId&&t.push(["rerun of",e.sourceJobId]),e.idempotentReplay&&t.push(["note","idempotent replay of an existing job"]),e.failure&&t.push(["failure",`${e.failure.code}: ${e.failure.message}`]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),f(t)}function gt(e){return [e.id,e.status,e.benchmark,String(e.trials.total),U(e.meanReward),C(e.spentUsd),e.createdAt]}function pt(e){return [e.taskKey,te(e.agent),String(e.runNumber),e.status,e.reward!==null?String(e.reward):"-",C(e.spentUsd),e.id]}function ft(e){let t=[["trial id",e.id],["job",e.jobId],["task",e.taskKey],["agent",te(e.agent)],["run",String(e.runNumber)],["status",e.status],["reward",e.reward!==null?String(e.reward):"-"]];return e.metrics&&Object.keys(e.metrics).length>0&&t.push(["metrics",Object.entries(e.metrics).map(([r,n])=>`${r}=${n}`).join(" \xB7 ")]),t.push(["spent",C(e.spentUsd)]),e.sandboxProvider&&t.push(["provider",e.sandboxProvider]),e.verifierMode&&t.push(["verifier",e.verifierMode]),e.resolvedHarnessVersion&&t.push(["harness version",e.resolvedHarnessVersion]),e.phaseTimingsMs&&Object.keys(e.phaseTimingsMs).length>0&&t.push(["timings",Object.entries(e.phaseTimingsMs).map(([r,n])=>`${r}=${n}ms`).join(" \xB7 ")]),e.failurePhase&&t.push(["failure phase",e.failurePhase]),e.failureDetail&&t.push(["failure detail",e.failureDetail]),e.sessionRef&&t.push(["session",e.sessionRef]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),f(t)}function bt(e){if(e===null)return "-";let t=Math.round(e*1e3)/1e3;return t>0?`+${t}`:String(t)}function ht(e){return [e.taskKey,e.status,U(e.sourceReward),U(e.reward),bt(e.rewardDelta),e.sourceTrialId]}function ye(e){let t=[["job id",e.id],["status",e.status],["source job",e.sourceJobId],["provider",e.sandboxProvider],["results",String(e.results.total)]],r=Object.entries(e.results.byStatus).filter(([,s])=>s>0).map(([s,o])=>`${s} ${o}`).join(" \xB7 ");if(r&&t.push(["by status",r]),e.filter&&(e.filter.status?.length||e.filter.taskKey)){let s=[];e.filter.status?.length&&s.push(`status=${e.filter.status.join(",")}`),e.filter.taskKey&&s.push(`task=${e.filter.taskKey}`),t.push(["filter",s.join(" \xB7 ")]);}t.push(["created",e.createdAt]);let n=f(t);if(e.results.items.length>0){n.push("");let s=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let o of e.results.items)s.push(ht(o));n.push(...f(s)),e.results.nextCursor&&n.push("",`More: evolve-evals regrade-job ${e.id} --cursor ${e.results.nextCursor}`);}return n}function be(e){let t=[["name",e.name],["source",e.source],["run command",e.runCommand]],r=Object.keys(e.env??{});return r.length>0&&t.push(["env",r.sort().join(", ")]),t.push(["created",e.createdAt]),t.push(["updated",e.updatedAt]),f(t)}var ke=["e2b","daytona","modal"];function yt(e){return ke.filter(t=>e?.[t]!==void 0).map(t=>`${t} ${e[t].ok?"ok":"NO"}`).join(" \xB7 ")}function U(e){return e!==null?String(Math.round(e*1e3)/1e3):"-"}function kt(e){let t=V(JSON.stringify(e.data??{}),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${t}`.trimEnd()}function Re(e){let t=e.failures?.length?` (${e.failures.length} task failure${e.failures.length===1?"":"s"})`:"";return `${e.message}${t}`}function ee(e){let t=[["id",e.id],["status",e.status]];if(e.benchmarkName!==void 0&&t.push(["benchmark",e.benchmarkName]),e.version!==void 0&&t.push(["version",e.version]),e.taskCount!==void 0&&t.push(["tasks",String(e.taskCount)]),e.failure){t.push(["failure",Re(e.failure)]);for(let r of e.failure.failures??[])t.push([`  ${r.taskKey}`,r.error]);}return f(t)}function Rt(e){let t=[];return e.taskCount!==void 0&&t.push(`tasks=${e.taskCount}`),e.failure&&t.push(V(Re(e.failure),140)),`status ${e.status.padEnd(12)} ${t.join(" ")}`.trimEnd()}function vt(e){let t={...e.data??{}},r=[];typeof t.trialId=="string"&&r.push(t.trialId),typeof t.taskKey=="string"&&r.push(t.taskKey),typeof t.status=="string"&&r.push(t.status),typeof t.reward=="number"&&r.push(`reward=${t.reward}`);let n=new Set(["trialId","taskKey","status","reward"]);for(let[o,a]of Object.entries(t))n.has(o)||r.push(`${o}=${typeof a=="object"?JSON.stringify(a):String(a)}`);let s=V(r.join(" "),140);return `#${String(e.seq).padStart(4)} ${e.type.padEnd(26)} ${s}`.trimEnd()}function b(e){let t={};return typeof e.flags["api-key"]=="string"&&(t.apiKey=e.flags["api-key"]),typeof e.flags["base-url"]=="string"&&(t.baseUrl=e.flags["base-url"]),t}function M(e){return {...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{}}}function St(e){return e.status==="COMPLETED"?0:e.status==="FAILED"||e.status==="CANCELLED"?1:0}async function wt(e,t){let r=ut(e),n=e.flags.json===true,s=e.flags.watch===true,o=h(b(e)),a=await o.run(r);if(!s){if(n)t.out(JSON.stringify(a));else {for(let u of _(a))t.out(u);t.out(""),t.out(`Follow it with: evolve-evals get ${a.id}`);}return 0}n?t.out(JSON.stringify({kind:"job.created",job:a})):t.out(`Job ${a.id} (${a.benchmark}) ${a.status} \u2014 watching\u2026`);let i=await o.watch(a.id,{onEvent:u=>{t.out(n?JSON.stringify({kind:"event",...u}):vt(u));}});if(n)t.out(JSON.stringify({kind:"job.final",job:i}));else {t.out("");for(let u of _(i))t.out(u);}return St(i)}async function Tt(e,t){let n=await h(b(e)).list({...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(n)),0;if(n.items.length===0)return t.out("No jobs."),0;let s=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let o of n.items)s.push(gt(o));for(let o of f(s))t.out(o);return n.nextCursor&&t.out(`
+More: evolve-evals list --cursor ${n.nextCursor}`),0}async function Ct(e,t){let n=await h(b(e)).get(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of _(n))t.out(s);return 0}async function It(e,t){let r=h(b(e)),n;if(e.flags.status!==void 0&&(n=String(e.flags.status).split(",").map(a=>a.trim()).filter(Boolean),n.length===0))throw new g("--status got an empty status list");let s=await r.trials(e.positionals[0],{...n!==void 0?{status:n}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{},...e.flags.cursor!==void 0?{cursor:e.flags.cursor}:{}});if(e.flags.json===true)return t.out(JSON.stringify(s)),0;if(s.items.length===0)return t.out("No trials."),0;let o=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let a of s.items)o.push(pt(a));for(let a of f(o))t.out(a);return t.out(`
+${s.items.length} trial(s) shown`),s.nextCursor&&t.out(`More: evolve-evals trials ${e.positionals[0]} --cursor ${s.nextCursor}`),0}async function xt(e,t){let n=await h(b(e)).trial(e.positionals[0],e.positionals[1]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of ft(n))t.out(s);return 0}async function Et(e,t){let r=h(b(e)),n=e.flags.json===true,s=0;for await(let o of r.trialTraceEvents(e.positionals[0],e.positionals[1],{...e.flags.cursor!==void 0?{cursor:String(e.flags.cursor)}:{},...e.flags.limit!==void 0?{limit:e.flags.limit}:{}}))t.out(n?JSON.stringify(o):kt(o)),s+=1;return !n&&s===0&&t.out("No trace events."),0}function Pt(e){let t=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let n of e.jobs)r.push([n.id,n.benchmark,n.status,U(n.meanReward),`${n.coverage.scored}/${n.coverage.total}`,C(n.spentUsd)]);if(t.push(...f(r)),e.taskMatrix.length>0){t.push("","Task matrix (disagreements first; columns in the order above):");let n=[["TASK","DIFF",...e.jobs.map((o,a)=>`JOB ${a+1}`)]],s=e.jobs.map(o=>o.id);for(let o of e.taskMatrix){let a=new Map(o.cells.map(i=>[i.jobId,i]));n.push([o.taskKey,o.disagreement?"!":"",...s.map(i=>{let u=a.get(i);return u?u.meanReward!==null?`${u.status} ${U(u.meanReward)}`:u.status:"-"})]);}t.push(...f(n));}return t}async function At(e,t){let n=await h(b(e)).compare(e.positionals);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of Pt(n))t.out(s);return 0}async function Ot(e,t){let n=await h(b(e)).cancel(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of _(n))t.out(s);return 0}async function Jt(e,t){let n=await h(b(e)).rerunFailed(e.positionals[0]);if(e.flags.json===true)t.out(JSON.stringify(n));else {for(let s of _(n))t.out(s);t.out(""),t.out(`Follow it with: evolve-evals get ${n.id}`);}return 0}async function _t(e,t){let r=h(b(e)),[n,s]=e.positionals,o;if(s!==void 0){if(e.flags.status!==void 0||e.flags.task!==void 0)throw new g("--status/--task apply to a whole-job regrade, not a single trial");o=await r.regradeTrial(n,s);}else {let a={};if(e.flags.status!==void 0){let i=String(e.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(i.length===0)throw new g("--status got an empty status list");a.status=i;}e.flags.task!==void 0&&(a.taskKey=String(e.flags.task)),o=await r.regrade(n,a);}if(e.flags.json===true)t.out(JSON.stringify(o));else {for(let a of ye(o))t.out(a);t.out(""),t.out(`Follow it with: evolve-evals regrade-job ${o.id}`);}return 0}async function Ut(e,t){let n=await h(b(e)).getRegrade(e.positionals[0],M(e));if(e.flags.json===true)t.out(JSON.stringify(n));else for(let s of ye(n))t.out(s);return 0}async function Lt(e,t){let r=e.flags.format;if(r!==void 0&&r!=="harbor")throw new g(`Unknown --format "${r}" (supported: harbor)`);let s=await h(b(e)).export(e.positionals[0],{to:e.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return e.flags.json===true?t.out(JSON.stringify({path:s})):t.out(`Saved ${s}`),0}function Dt(e){let t=f([["name",e.name],["title",e.title??"-"],["description",e.description??"-"],["active version",e.activeVersion?.version??"-"]]);if(e.versions&&e.versions.length>0){t.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let n of e.versions)r.push([n.version,n.state,String(n.taskCount),n.createdAt??"-"]);t.push(...f(r));}if(e.tasks&&e.tasks.items.length>0){t.push("",`Tasks (version ${e.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let s of e.tasks.items)r.push([s.taskKey,`${s.agentTimeoutSec}s`,`${s.verifierTimeoutSec}s`,yt(s.providers)]);t.push(...f(r)),e.tasks.nextCursor&&t.push(`More tasks: evolve-evals benchmarks get ${e.name} --cursor ${e.tasks.nextCursor}`);let n=new Map;for(let s of e.tasks.items)for(let o of ke){let a=s.providers?.[o];a&&!a.ok&&!n.has(`${o}:${a.reason}`)&&n.set(`${o}:${a.reason}`,`${o}: ${a.reason}`);}if(n.size>0){t.push("","Provider limitations:");for(let s of n.values())t.push(`  ${s}`);}}return t}async function $t(e,t){let r=Z(b(e)),[n,s]=e.positionals;if(n===void 0){let a=await r.list(M(e));if(e.flags.json===true)return t.out(JSON.stringify(a)),0;if(a.items.length===0)return t.out("No benchmarks."),0;let i=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let u of a.items)i.push([u.name,u.activeVersion?.version??"-",u.activeVersion?.state??"-",u.activeVersion?String(u.activeVersion.taskCount):"-",u.title??"-"]);for(let u of f(i))t.out(u);for(let u of he(a.items))t.out(u);return 0}if(n!=="get")throw new g(`Unknown benchmarks subcommand "${n}" (did you mean "benchmarks get ${n}"?)`);if(!s)throw new g('"benchmarks get" requires a <name[@version]> ref');let o=await r.get(s,M(e));if(e.flags.json===true)t.out(JSON.stringify(o));else {for(let a of Dt(o))t.out(a);for(let a of he([o]))t.out(a);}return 0}function he(e){let t=[];for(let r of e){if(!r.upstream?.moved)continue;let n=r.activeVersion?`@${r.activeVersion.version}`:"";t.push(`${r.name}${n} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return t}async function Bt(e,t){let[r,n]=e.positionals,s=e.flags.json===true,o=Z(b(e));if(r==="status"){if(!n)throw new g('"import status" requires an <id>');let l=await o.getImport(n);if(s)t.out(JSON.stringify(l));else for(let c of ee(l))t.out(c);return 0}if(r!==void 0)throw new g(`Unknown import subcommand "${r}" (supported: status)`);let a=lt(e),i=await o.import(a);if(e.flags.watch!==true){if(s)t.out(JSON.stringify(i));else {for(let l of ee(i))t.out(l);t.out(""),t.out(`Follow it with: evolve-evals import status ${i.id}`);}return 0}s?t.out(JSON.stringify({kind:"import.created",benchmarkImport:i})):t.out(`Import ${i.id} (${a.benchmarkName}) ${i.status} \u2014 watching\u2026`);let u=await o.watchImport(i.id,{onStatus:l=>{t.out(s?JSON.stringify({kind:"import.status",benchmarkImport:l}):Rt(l));}});if(s)t.out(JSON.stringify({kind:"import.final",benchmarkImport:u}));else {t.out("");for(let l of ee(u))t.out(l);}return u.status==="FAILED"?1:0}async function Nt(e,t){let r=pe(b(e)),[n,s]=e.positionals,o=e.flags.json===true;if(n===void 0){let a=await r.list(M(e));if(o)return t.out(JSON.stringify(a)),0;if(a.items.length===0)return t.out("No custom harnesses."),0;let i=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let u of a.items)i.push([u.name,u.source,V(u.runCommand,60),u.updatedAt]);for(let u of f(i))t.out(u);return 0}if(n==="add"){if(s!==void 0)throw new g(`"custom-harnesses add" got unexpected argument "${s}"`);let a=await r.create(dt(e));if(o)t.out(JSON.stringify(a));else {for(let i of be(a))t.out(i);t.out(""),t.out(`Use it with: evolve-evals run --agent ${a.name}:<model> \u2026`);}return 0}if(n==="get"){if(!s)throw new g('"custom-harnesses get" requires a <name>');let a=await r.get(s);if(o)t.out(JSON.stringify(a));else for(let i of be(a))t.out(i);return 0}if(n==="remove"){if(!s)throw new g('"custom-harnesses remove" requires a <name>');return await r.delete(s),o?t.out(JSON.stringify({name:s,deleted:true})):t.out(`Deleted custom harness ${s}`),0}throw new g(`Unknown custom-harnesses subcommand "${n}" (supported: add, get, remove)`)}async function jt(e,t=mt){let r;try{r=ot(e);}catch(n){if(n instanceof g)return t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2;throw n}try{switch(r.command){case "help":return t.out(nt),0;case "run":return await wt(r,t);case "list":return await Tt(r,t);case "get":return await Ct(r,t);case "trials":return await It(r,t);case "trial":return await xt(r,t);case "trace":return await Et(r,t);case "compare":return await At(r,t);case "cancel":return await Ot(r,t);case "rerun-failed":return await Jt(r,t);case "regrade":return await _t(r,t);case "regrade-job":return await Ut(r,t);case "export":return await Lt(r,t);case "benchmarks":return await $t(r,t);case "import":return await Bt(r,t);case "custom-harnesses":return await Nt(r,t);default:return t.err(`Error: unknown command "${r.command}"`),2}}catch(n){return n instanceof g?(t.err(`Error: ${n.message}`),t.err('Run "evolve-evals help" for usage.'),2):(t.err(`Error: ${n.message}`),1)}}var Ht=(()=>{let e=process.argv[1];if(!e)return  false;try{return (typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('cli.cjs', document.baseURI).href))===url.pathToFileURL(e).href}catch{return  false}})();Ht&&jt(process.argv.slice(2)).then(e=>{process.exitCode=e;},e=>{process.stderr.write(`Error: ${e?.message??e}
+`),process.exitCode=1;});exports.CliUsageError=g;exports.USAGE=nt;exports.buildCustomHarnessInput=dt;exports.buildImportInput=lt;exports.buildJobInput=ut;exports.eventLine=vt;exports.importStatusLine=Rt;exports.parseArgs=ot;exports.parseHarnessEnv=ct;exports.parseJobAgent=it;exports.runCli=jt;exports.traceEventLine=kt;

package/dist/hosted/cli.d.cts

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BeJrn1lR.cjs';
+
+/**
+ * evolve-evals — CLI for Evolve hosted benchmarks & jobs.
+ *
+ * Thin shell over the hosted client (benchmarks() / jobs()): plain node
+ * arg parsing, no dependencies. Human-readable tables by default; --json emits
+ * machine-readable JSON (NDJSON for --watch event streams).
+ *
+ * Exit codes: 0 success (watch: job COMPLETED / import IMPORTED), 1
+ * runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
+ */
+
+declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n  run                               Create a job (add --watch to follow it)\n  list                              List your jobs (newest first)\n  get <id>                          Show one job\n  trials <id>                       List a job's trials\n  trial <id> <trial-id>             Show one trial in full detail\n  trace <id> <trial-id>             Print a trial's trace events\n  compare <id> <id> [...]           Compare 2-5 jobs side by side\n  cancel <id>                       Request cancellation of a job\n  rerun-failed <id>                 New job from a terminal job's failed trials\n  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)\n  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)\n  export <id>                       Download the research archive (gzipped JSON)\n  benchmarks                        List the benchmark catalog\n  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)\n  import                            Import a benchmark from a git source or a local directory (--watch to follow)\n  import status <id>                Show one import job\n  custom-harnesses                  List your registered custom harnesses\n  custom-harnesses get <name>       Show one custom harness\n  custom-harnesses add              Register a custom harness (install script or local directory)\n  custom-harnesses remove <name>    Delete a custom harness\n  help                              Show this help\n\nRun options:\n  --benchmark <name[@version]>        Benchmark (required; bare name = active version)\n  --tasks <k1,k2,...>                 Task keys (default: every task of the version)\n  --agent <harness:model[:version]>   Agent; repeatable (at least one required)\n  --effort <value>                    Reasoning effort for EVERY agent (values:\n                                      GET /api/meta limits.job.reasoningEfforts).\n                                      Applied verbatim \u2014 an agent whose harness\n                                      cannot honor it is refused by the server,\n                                      never silently skipped. Per-agent efforts\n                                      need the SDK. Omitted: the server default.\n  --runs <n>                          Runs per task x agent (default 1)\n  --concurrency <n>                   Parallel trials (default 1)\n  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)\n  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b\n  --watch                             Stream events until the job finishes\n\nTrial options:\n  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n  --status <s1,s2,...>                Only regrade source trials in these statuses\n  --task <key>                        Only regrade source trials of this task\n\nTrace options:\n  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)\n  --limit <n>                         Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n  --git <url>                         Git repository URL (with --ref)\n  --ref <ref>                         Git ref: branch, tag, or commit (with --git)\n  --dir <path>                        Local corpus directory (tarred + uploaded)\n  --name <benchmark>                  Catalog benchmark name to create or extend (required)\n  --version <v>                       Version label for the imported version (required)\n  --watch                             Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n  --name <harness>                    Harness name, later used in --agent (required)\n  --install-script <path>             Install script file; its contents are uploaded\n  --dir <path>                        Local harness directory (tarred + uploaded)\n  --run <command>                     Run command, executed with sh -c (required)\n  --env KEY=VALUE                     Env injected at run time; repeatable\n\nOther options:\n  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection\n                                      (list, trials, trace, benchmarks, benchmarks get,\n                                      custom-harnesses, regrade-job)\n  --to <dir>                          Export target directory (default: current dir)\n  --format harbor                     Export the Harbor job-layout bundle\n  --json                              Machine-readable JSON output\n  --api-key <key>                     API key (default: $EVOLVE_API_KEY)\n  --base-url <url>                    API base URL (default: the Evolve dashboard API)";
+/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
+declare class CliUsageError extends Error {
+    constructor(message: string);
+}
+interface Invocation {
+    command: string;
+    positionals: string[];
+    flags: Record<string, string | number | boolean | string[]>;
+}
+declare function parseArgs(argv: string[]): Invocation;
+/** Parse "harness:model[:version]" (version may itself contain colons). */
+declare function parseJobAgent(spec: string): JobAgent;
+/**
+ * Build the POST /api/jobs body from a parsed `run` invocation.
+ * Keys follow the contract field order: benchmark, tasks, agents,
+ * runsPerTask, concurrency, maxTrialSpendUsd, sandboxProvider.
+ */
+declare function buildJobInput(inv: Invocation): JobInput;
+/** Build the benchmarks().import() input from a parsed `import` invocation. */
+declare function buildImportInput(inv: Invocation): BenchmarkImportInput;
+/** Parse repeatable `--env KEY=VALUE` into the declared run-time env map. */
+declare function parseHarnessEnv(pairs: string[]): Record<string, string>;
+/**
+ * Build the customHarnesses().create() input from a parsed
+ * `custom-harnesses add` invocation. `--install-script` names a FILE; its
+ * contents are what the SDK uploads.
+ */
+declare function buildCustomHarnessInput(inv: Invocation, readScript?: (path: string) => string): CustomHarnessInput;
+interface CliIO {
+    out(line: string): void;
+    err(line: string): void;
+}
+/** One trace event line — evolve-evals trace. */
+declare function traceEventLine(event: TrialTraceEvent): string;
+/** Compact one-line rendering of one import status change for --watch. */
+declare function importStatusLine(job: BenchmarkImport): string;
+/** Compact one-line rendering of one SSE event for --watch. */
+declare function eventLine(event: JobEvent): string;
+declare function runCli(argv: string[], io?: CliIO): Promise<number>;
+
+export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };

package/dist/hosted/cli.d.ts

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+import { o as JobAgent, p as JobInput, Z as BenchmarkImportInput, a2 as CustomHarnessInput, w as TrialTraceEvent, Y as BenchmarkImport, y as JobEvent } from '../types-BeJrn1lR.js';
+
+/**
+ * evolve-evals — CLI for Evolve hosted benchmarks & jobs.
+ *
+ * Thin shell over the hosted client (benchmarks() / jobs()): plain node
+ * arg parsing, no dependencies. Human-readable tables by default; --json emits
+ * machine-readable JSON (NDJSON for --watch event streams).
+ *
+ * Exit codes: 0 success (watch: job COMPLETED / import IMPORTED), 1
+ * runtime/API failure (watch: FAILED or CANCELLED), 2 usage error.
+ */
+
+declare const USAGE = "evolve-evals \u2014 Evolve hosted jobs CLI\n\nUsage: evolve-evals <command> [options]\n\nCommands:\n  run                               Create a job (add --watch to follow it)\n  list                              List your jobs (newest first)\n  get <id>                          Show one job\n  trials <id>                       List a job's trials\n  trial <id> <trial-id>             Show one trial in full detail\n  trace <id> <trial-id>             Print a trial's trace events\n  compare <id> <id> [...]           Compare 2-5 jobs side by side\n  cancel <id>                       Request cancellation of a job\n  rerun-failed <id>                 New job from a terminal job's failed trials\n  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)\n  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)\n  export <id>                       Download the research archive (gzipped JSON)\n  benchmarks                        List the benchmark catalog\n  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)\n  import                            Import a benchmark from a git source or a local directory (--watch to follow)\n  import status <id>                Show one import job\n  custom-harnesses                  List your registered custom harnesses\n  custom-harnesses get <name>       Show one custom harness\n  custom-harnesses add              Register a custom harness (install script or local directory)\n  custom-harnesses remove <name>    Delete a custom harness\n  help                              Show this help\n\nRun options:\n  --benchmark <name[@version]>        Benchmark (required; bare name = active version)\n  --tasks <k1,k2,...>                 Task keys (default: every task of the version)\n  --agent <harness:model[:version]>   Agent; repeatable (at least one required)\n  --effort <value>                    Reasoning effort for EVERY agent (values:\n                                      GET /api/meta limits.job.reasoningEfforts).\n                                      Applied verbatim \u2014 an agent whose harness\n                                      cannot honor it is refused by the server,\n                                      never silently skipped. Per-agent efforts\n                                      need the SDK. Omitted: the server default.\n  --runs <n>                          Runs per task x agent (default 1)\n  --concurrency <n>                   Parallel trials (default 1)\n  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)\n  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b\n  --watch                             Stream events until the job finishes\n\nTrial options:\n  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)\n\nRegrade options (whole-job regrade only):\n  --status <s1,s2,...>                Only regrade source trials in these statuses\n  --task <key>                        Only regrade source trials of this task\n\nTrace options:\n  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)\n  --limit <n>                         Max events per page\n\nImport options (a git source OR a local directory; --name and --version required):\n  --git <url>                         Git repository URL (with --ref)\n  --ref <ref>                         Git ref: branch, tag, or commit (with --git)\n  --dir <path>                        Local corpus directory (tarred + uploaded)\n  --name <benchmark>                  Catalog benchmark name to create or extend (required)\n  --version <v>                       Version label for the imported version (required)\n  --watch                             Poll until the import is IMPORTED or FAILED\n\nCustom-harness options (\"custom-harnesses add\"; an install script OR a local directory):\n  --name <harness>                    Harness name, later used in --agent (required)\n  --install-script <path>             Install script file; its contents are uploaded\n  --dir <path>                        Local harness directory (tarred + uploaded)\n  --run <command>                     Run command, executed with sh -c (required)\n  --env KEY=VALUE                     Env injected at run time; repeatable\n\nOther options:\n  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection\n                                      (list, trials, trace, benchmarks, benchmarks get,\n                                      custom-harnesses, regrade-job)\n  --to <dir>                          Export target directory (default: current dir)\n  --format harbor                     Export the Harbor job-layout bundle\n  --json                              Machine-readable JSON output\n  --api-key <key>                     API key (default: $EVOLVE_API_KEY)\n  --base-url <url>                    API base URL (default: the Evolve dashboard API)";
+/** Usage-level error: bad command line, not a runtime failure. Exit code 2. */
+declare class CliUsageError extends Error {
+    constructor(message: string);
+}
+interface Invocation {
+    command: string;
+    positionals: string[];
+    flags: Record<string, string | number | boolean | string[]>;
+}
+declare function parseArgs(argv: string[]): Invocation;
+/** Parse "harness:model[:version]" (version may itself contain colons). */
+declare function parseJobAgent(spec: string): JobAgent;
+/**
+ * Build the POST /api/jobs body from a parsed `run` invocation.
+ * Keys follow the contract field order: benchmark, tasks, agents,
+ * runsPerTask, concurrency, maxTrialSpendUsd, sandboxProvider.
+ */
+declare function buildJobInput(inv: Invocation): JobInput;
+/** Build the benchmarks().import() input from a parsed `import` invocation. */
+declare function buildImportInput(inv: Invocation): BenchmarkImportInput;
+/** Parse repeatable `--env KEY=VALUE` into the declared run-time env map. */
+declare function parseHarnessEnv(pairs: string[]): Record<string, string>;
+/**
+ * Build the customHarnesses().create() input from a parsed
+ * `custom-harnesses add` invocation. `--install-script` names a FILE; its
+ * contents are what the SDK uploads.
+ */
+declare function buildCustomHarnessInput(inv: Invocation, readScript?: (path: string) => string): CustomHarnessInput;
+interface CliIO {
+    out(line: string): void;
+    err(line: string): void;
+}
+/** One trace event line — evolve-evals trace. */
+declare function traceEventLine(event: TrialTraceEvent): string;
+/** Compact one-line rendering of one import status change for --watch. */
+declare function importStatusLine(job: BenchmarkImport): string;
+/** Compact one-line rendering of one SSE event for --watch. */
+declare function eventLine(event: JobEvent): string;
+declare function runCli(argv: string[], io?: CliIO): Promise<number>;
+
+export { type CliIO, CliUsageError, type Invocation, USAGE, buildCustomHarnessInput, buildImportInput, buildJobInput, eventLine, importStatusLine, parseArgs, parseHarnessEnv, parseJobAgent, runCli, traceEventLine };

package/dist/hosted/cli.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+import {G as G$1,F as F$1,H as H$1}from'../chunk-IHBQMTA4.js';import {readFileSync}from'fs';import {pathToFileURL}from'url';var j=`evolve-evals \u2014 Evolve hosted jobs CLI
+
+Usage: evolve-evals <command> [options]
+
+Commands:
+  run                               Create a job (add --watch to follow it)
+  list                              List your jobs (newest first)
+  get <id>                          Show one job
+  trials <id>                       List a job's trials
+  trial <id> <trial-id>             Show one trial in full detail
+  trace <id> <trial-id>             Print a trial's trace events
+  compare <id> <id> [...]           Compare 2-5 jobs side by side
+  cancel <id>                       Request cancellation of a job
+  rerun-failed <id>                 New job from a terminal job's failed trials
+  regrade <id> [trial-id]           Re-run the verifier on recorded trials (whole job, or one trial)
+  regrade-job <job-id>              Show a regrade job's results (rewards, deltas, lineage)
+  export <id>                       Download the research archive (gzipped JSON)
+  benchmarks                        List the benchmark catalog
+  benchmarks get <name[@version]>   Show one benchmark (versions + tasks + providers)
+  import                            Import a benchmark from a git source or a local directory (--watch to follow)
+  import status <id>                Show one import job
+  custom-harnesses                  List your registered custom harnesses
+  custom-harnesses get <name>       Show one custom harness
+  custom-harnesses add              Register a custom harness (install script or local directory)
+  custom-harnesses remove <name>    Delete a custom harness
+  help                              Show this help
+
+Run options:
+  --benchmark <name[@version]>        Benchmark (required; bare name = active version)
+  --tasks <k1,k2,...>                 Task keys (default: every task of the version)
+  --agent <harness:model[:version]>   Agent; repeatable (at least one required)
+  --effort <value>                    Reasoning effort for EVERY agent (values:
+                                      GET /api/meta limits.job.reasoningEfforts).
+                                      Applied verbatim \u2014 an agent whose harness
+                                      cannot honor it is refused by the server,
+                                      never silently skipped. Per-agent efforts
+                                      need the SDK. Omitted: the server default.
+  --runs <n>                          Runs per task x agent (default 1)
+  --concurrency <n>                   Parallel trials (default 1)
+  --max-trial-spend <usd>             Model-spend cap for EACH trial (default: the server's, $200)
+  --provider <e2b|daytona|modal>      e2b | daytona | modal, default e2b
+  --watch                             Stream events until the job finishes
+
+Trial options:
+  --status <s1,s2,...>                Filter trials by status (e.g. INFRASTRUCTURE_ERROR)
+
+Regrade options (whole-job regrade only):
+  --status <s1,s2,...>                Only regrade source trials in these statuses
+  --task <key>                        Only regrade source trials of this task
+
+Trace options:
+  --cursor <seq>                      Resume after this trace seq (a trace cursor IS a seq)
+  --limit <n>                         Max events per page
+
+Import options (a git source OR a local directory; --name and --version required):
+  --git <url>                         Git repository URL (with --ref)
+  --ref <ref>                         Git ref: branch, tag, or commit (with --git)
+  --dir <path>                        Local corpus directory (tarred + uploaded)
+  --name <benchmark>                  Catalog benchmark name to create or extend (required)
+  --version <v>                       Version label for the imported version (required)
+  --watch                             Poll until the import is IMPORTED or FAILED
+
+Custom-harness options ("custom-harnesses add"; an install script OR a local directory):
+  --name <harness>                    Harness name, later used in --agent (required)
+  --install-script <path>             Install script file; its contents are uploaded
+  --dir <path>                        Local harness directory (tarred + uploaded)
+  --run <command>                     Run command, executed with sh -c (required)
+  --env KEY=VALUE                     Env injected at run time; repeatable
+
+Other options:
+  --limit <n>, --cursor <c>           Pagination \u2014 one envelope on every collection
+                                      (list, trials, trace, benchmarks, benchmarks get,
+                                      custom-harnesses, regrade-job)
+  --to <dir>                          Export target directory (default: current dir)
+  --format harbor                     Export the Harbor job-layout bundle
+  --json                              Machine-readable JSON output
+  --api-key <key>                     API key (default: $EVOLVE_API_KEY)
+  --base-url <url>                    API base URL (default: the Evolve dashboard API)`,l=class extends Error{constructor(e){super(e),this.name="CliUsageError";}},J={json:"boolean","api-key":"string","base-url":"string"},U={run:{flags:{benchmark:"string",tasks:"string",agent:"repeat",effort:"string",runs:"number",concurrency:"number","max-trial-spend":"number",provider:"string",watch:"boolean"},required:["benchmark","agent"],minPositionals:0,maxPositionals:0},list:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:0},get:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trials:{flags:{status:"string",limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},trial:{flags:{},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},trace:{flags:{cursor:"string",limit:"number"},minPositionals:2,maxPositionals:2,positionalUsage:"<id> <trial-id>"},compare:{flags:{},minPositionals:2,maxPositionals:5,positionalUsage:"<id> <id> [...]"},cancel:{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},"rerun-failed":{flags:{},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},regrade:{flags:{status:"string",task:"string"},minPositionals:1,maxPositionals:2,positionalUsage:"<id> [trial-id]"},"regrade-job":{flags:{limit:"number",cursor:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<job-id>"},export:{flags:{to:"string",format:"string"},minPositionals:1,maxPositionals:1,positionalUsage:"<id>"},benchmarks:{flags:{limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},import:{flags:{git:"string",ref:"string",dir:"string",name:"string",version:"string",watch:"boolean"},minPositionals:0,maxPositionals:2},"custom-harnesses":{flags:{name:"string","install-script":"string",dir:"string",run:"string",env:"repeat",limit:"number",cursor:"string"},minPositionals:0,maxPositionals:2},help:{flags:{},minPositionals:0,maxPositionals:0}},O={command:"help",positionals:[],flags:{}};function L(t){if(t.length===0)throw new l("No command given");if(t[0]==="--help"||t[0]==="-h")return O;let e=t[0],r=U[e];if(!r)throw new l(`Unknown command "${e}"`);let s={},n=[];for(let o=1;o<t.length;o++){let i=t[o];if(i==="--help"||i==="-h")return O;if(!i.startsWith("--")){n.push(i);continue}let a=i.slice(2),u,d=a.indexOf("=");d!==-1&&(u=a.slice(d+1),a=a.slice(0,d));let w=r.flags[a]??J[a];if(!w)throw new l(`Unknown option --${a} for "${e}"`);if(w==="boolean"){if(u!==void 0)throw new l(`Option --${a} takes no value`);s[a]=true;continue}let p=u;if(p===void 0){let g=t[o+1];if(g===void 0||g.startsWith("--"))throw new l(`Option --${a} requires a value`);p=g,o++;}if(w==="number"){let g=Number(p);if(p.trim()===""||!Number.isFinite(g))throw new l(`Option --${a} expects a number, got "${p}"`);s[a]=g;}else if(w==="repeat"){let g=s[a]??[];g.push(p),s[a]=g;}else s[a]=p;}if(n.length<r.minPositionals)throw new l(`"${e}" requires ${r.positionalUsage??`${r.minPositionals} argument(s)`}`);if(n.length>r.maxPositionals)throw new l(`"${e}" got unexpected argument "${n[r.maxPositionals]}"`);for(let o of r.required??[])if(!(o in s))throw new l(`"${e}" requires --${o}`);return {command:e,positionals:n,flags:s}}function D(t){let e=t.indexOf(":");if(e<=0||e===t.length-1)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);let r=t.slice(0,e),s=t.slice(e+1),n=s.indexOf(":");if(n===-1)return {harness:r,model:s};let o=s.slice(0,n),i=s.slice(n+1);if(!o||!i)throw new l(`Invalid --agent "${t}": expected harness:model[:version]`);return {harness:r,model:o,harnessVersion:i}}function q(t){let e=t.flags,r;if(e.tasks!==void 0&&(r=String(e.tasks).split(",").map(s=>s.trim()).filter(Boolean),r.length===0))throw new l("--tasks got an empty task list");return {benchmark:e.benchmark,...r!==void 0?{tasks:r}:{},agents:e.agent.map(s=>{let n=D(s);return e.effort!==void 0?{...n,reasoningEffort:e.effort}:n}),...e.runs!==void 0?{runsPerTask:e.runs}:{},...e.concurrency!==void 0?{concurrency:e.concurrency}:{},...e["max-trial-spend"]!==void 0?{maxTrialSpendUsd:e["max-trial-spend"]}:{},...e.provider!==void 0?{sandboxProvider:e.provider}:{}}}function M(t){let e=t.flags,r=typeof e.dir=="string",s=typeof e.git=="string"||typeof e.ref=="string";if(r&&s)throw new l('"import" takes EITHER --dir OR --git/--ref, not both');if(r){for(let n of ["name","version"])if(typeof e[n]!="string")throw new l(`"import" requires --${n}`);return {source:{directory:e.dir},benchmarkName:e.name,version:e.version}}for(let n of ["git","ref","name","version"])if(typeof e[n]!="string"){let o=n==="git"||n==="ref"?" (or --dir for a local corpus directory)":"";throw new l(`"import" requires --${n}${o}`)}return {source:{gitUrl:e.git,ref:e.ref},benchmarkName:e.name,version:e.version}}function K(t){let e={};for(let r of t){let s=r.indexOf("=");if(s<=0)throw new l(`Invalid --env "${r}": expected KEY=VALUE`);e[r.slice(0,s)]=r.slice(s+1);}return e}function V(t,e=r=>readFileSync(r,"utf-8")){let r=t.flags,s=typeof r.dir=="string",n=typeof r["install-script"]=="string";if(s&&n)throw new l('"custom-harnesses add" takes EITHER --dir OR --install-script, not both');if(!s&&!n)throw new l('"custom-harnesses add" requires --install-script (or --dir for a local harness directory)');for(let i of ["name","run"])if(typeof r[i]!="string")throw new l(`"custom-harnesses add" requires --${i}`);let o=K(r.env??[]);return {name:r.name,...s?{directory:r.dir}:{installScript:e(r["install-script"])},runCommand:r.run,...Object.keys(o).length>0?{env:o}:{}}}var F={out:t=>process.stdout.write(t+`
+`),err:t=>process.stderr.write(t+`
+`)};function c(t){if(t.length===0)return [];let e=[];for(let r of t)r.forEach((s,n)=>{e[n]=Math.max(e[n]??0,s.length);});return t.map(r=>r.map((s,n)=>s.padEnd(e[n])).join("  ").trimEnd())}function h(t){return typeof t=="number"?`$${t.toFixed(2)}`:"-"}function $(t){let e=`${t.harness}:${t.model}`;return t.harnessVersion?`${e}:${t.harnessVersion}`:e}function v(t,e){return t.length>e?t.slice(0,e-1)+"\u2026":t}function b(t){let e=[["id",t.id],["status",t.status],["benchmark",t.benchmark]];e.push(["agents",t.agents.map($).join(", ")]),e.push(["size",`${t.counts.agents} agent(s) x ${t.counts.tasks} task(s) = ${t.trials.total} trial(s)`]),e.push(["runs/task",String(t.runsPerTask)]),e.push(["concurrency",String(t.concurrency)]),e.push(["max spend/trial",h(t.maxTrialSpendUsd)]),e.push(["worst case",h(t.worstCaseSpendUsd)]),e.push(["provider",t.sandboxProvider]),e.push(["spent",h(t.spentUsd)]),e.push(["mean reward",t.meanReward!==null?String(t.meanReward):"-"]);let r=Object.entries(t.trials.byStatus).filter(([,s])=>s>0).map(([s,n])=>`${s} ${n}`).join(" \xB7 ");return r&&e.push(["trials",r]),t.sourceJobId&&e.push(["rerun of",t.sourceJobId]),t.idempotentReplay&&e.push(["note","idempotent replay of an existing job"]),t.failure&&e.push(["failure",`${t.failure.code}: ${t.failure.message}`]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),c(e)}function H(t){return [t.id,t.status,t.benchmark,String(t.trials.total),k(t.meanReward),h(t.spentUsd),t.createdAt]}function B(t){return [t.taskKey,$(t.agent),String(t.runNumber),t.status,t.reward!==null?String(t.reward):"-",h(t.spentUsd),t.id]}function G(t){let e=[["trial id",t.id],["job",t.jobId],["task",t.taskKey],["agent",$(t.agent)],["run",String(t.runNumber)],["status",t.status],["reward",t.reward!==null?String(t.reward):"-"]];return t.metrics&&Object.keys(t.metrics).length>0&&e.push(["metrics",Object.entries(t.metrics).map(([r,s])=>`${r}=${s}`).join(" \xB7 ")]),e.push(["spent",h(t.spentUsd)]),t.sandboxProvider&&e.push(["provider",t.sandboxProvider]),t.verifierMode&&e.push(["verifier",t.verifierMode]),t.resolvedHarnessVersion&&e.push(["harness version",t.resolvedHarnessVersion]),t.phaseTimingsMs&&Object.keys(t.phaseTimingsMs).length>0&&e.push(["timings",Object.entries(t.phaseTimingsMs).map(([r,s])=>`${r}=${s}ms`).join(" \xB7 ")]),t.failurePhase&&e.push(["failure phase",t.failurePhase]),t.failureDetail&&e.push(["failure detail",t.failureDetail]),t.sessionRef&&e.push(["session",t.sessionRef]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),c(e)}function _(t){if(t===null)return "-";let e=Math.round(t*1e3)/1e3;return e>0?`+${e}`:String(e)}function W(t){return [t.taskKey,t.status,k(t.sourceReward),k(t.reward),_(t.rewardDelta),t.sourceTrialId]}function P(t){let e=[["job id",t.id],["status",t.status],["source job",t.sourceJobId],["provider",t.sandboxProvider],["results",String(t.results.total)]],r=Object.entries(t.results.byStatus).filter(([,n])=>n>0).map(([n,o])=>`${n} ${o}`).join(" \xB7 ");if(r&&e.push(["by status",r]),t.filter&&(t.filter.status?.length||t.filter.taskKey)){let n=[];t.filter.status?.length&&n.push(`status=${t.filter.status.join(",")}`),t.filter.taskKey&&n.push(`task=${t.filter.taskKey}`),e.push(["filter",n.join(" \xB7 ")]);}e.push(["created",t.createdAt]);let s=c(e);if(t.results.items.length>0){s.push("");let n=[["TASK","STATUS","WAS","NOW","\u0394","SOURCE TRIAL ID"]];for(let o of t.results.items)n.push(W(o));s.push(...c(n)),t.results.nextCursor&&s.push("",`More: evolve-evals regrade-job ${t.id} --cursor ${t.results.nextCursor}`);}return s}function R(t){let e=[["name",t.name],["source",t.source],["run command",t.runCommand]],r=Object.keys(t.env??{});return r.length>0&&e.push(["env",r.sort().join(", ")]),e.push(["created",t.createdAt]),e.push(["updated",t.updatedAt]),c(e)}var C=["e2b","daytona","modal"];function Y(t){return C.filter(e=>t?.[e]!==void 0).map(e=>`${e} ${t[e].ok?"ok":"NO"}`).join(" \xB7 ")}function k(t){return t!==null?String(Math.round(t*1e3)/1e3):"-"}function z(t){let e=v(JSON.stringify(t.data??{}),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${e}`.trimEnd()}function T(t){let e=t.failures?.length?` (${t.failures.length} task failure${t.failures.length===1?"":"s"})`:"";return `${t.message}${e}`}function I(t){let e=[["id",t.id],["status",t.status]];if(t.benchmarkName!==void 0&&e.push(["benchmark",t.benchmarkName]),t.version!==void 0&&e.push(["version",t.version]),t.taskCount!==void 0&&e.push(["tasks",String(t.taskCount)]),t.failure){e.push(["failure",T(t.failure)]);for(let r of t.failure.failures??[])e.push([`  ${r.taskKey}`,r.error]);}return c(e)}function Q(t){let e=[];return t.taskCount!==void 0&&e.push(`tasks=${t.taskCount}`),t.failure&&e.push(v(T(t.failure),140)),`status ${t.status.padEnd(12)} ${e.join(" ")}`.trimEnd()}function X(t){let e={...t.data??{}},r=[];typeof e.trialId=="string"&&r.push(e.trialId),typeof e.taskKey=="string"&&r.push(e.taskKey),typeof e.status=="string"&&r.push(e.status),typeof e.reward=="number"&&r.push(`reward=${e.reward}`);let s=new Set(["trialId","taskKey","status","reward"]);for(let[o,i]of Object.entries(e))s.has(o)||r.push(`${o}=${typeof i=="object"?JSON.stringify(i):String(i)}`);let n=v(r.join(" "),140);return `#${String(t.seq).padStart(4)} ${t.type.padEnd(26)} ${n}`.trimEnd()}function f(t){let e={};return typeof t.flags["api-key"]=="string"&&(e.apiKey=t.flags["api-key"]),typeof t.flags["base-url"]=="string"&&(e.baseUrl=t.flags["base-url"]),e}function y(t){return {...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{}}}function Z(t){return t.status==="COMPLETED"?0:t.status==="FAILED"||t.status==="CANCELLED"?1:0}async function tt(t,e){let r=q(t),s=t.flags.json===true,n=t.flags.watch===true,o=H$1(f(t)),i=await o.run(r);if(!n){if(s)e.out(JSON.stringify(i));else {for(let u of b(i))e.out(u);e.out(""),e.out(`Follow it with: evolve-evals get ${i.id}`);}return 0}s?e.out(JSON.stringify({kind:"job.created",job:i})):e.out(`Job ${i.id} (${i.benchmark}) ${i.status} \u2014 watching\u2026`);let a=await o.watch(i.id,{onEvent:u=>{e.out(s?JSON.stringify({kind:"event",...u}):X(u));}});if(s)e.out(JSON.stringify({kind:"job.final",job:a}));else {e.out("");for(let u of b(a))e.out(u);}return Z(a)}async function et(t,e){let s=await H$1(f(t)).list({...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(s)),0;if(s.items.length===0)return e.out("No jobs."),0;let n=[["ID","STATUS","BENCHMARK","TRIALS","MEAN REWARD","SPENT","CREATED"]];for(let o of s.items)n.push(H(o));for(let o of c(n))e.out(o);return s.nextCursor&&e.out(`
+More: evolve-evals list --cursor ${s.nextCursor}`),0}async function st(t,e){let s=await H$1(f(t)).get(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let n of b(s))e.out(n);return 0}async function rt(t,e){let r=H$1(f(t)),s;if(t.flags.status!==void 0&&(s=String(t.flags.status).split(",").map(i=>i.trim()).filter(Boolean),s.length===0))throw new l("--status got an empty status list");let n=await r.trials(t.positionals[0],{...s!==void 0?{status:s}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{},...t.flags.cursor!==void 0?{cursor:t.flags.cursor}:{}});if(t.flags.json===true)return e.out(JSON.stringify(n)),0;if(n.items.length===0)return e.out("No trials."),0;let o=[["TASK","AGENT","RUN","STATUS","REWARD","SPENT","TRIAL ID"]];for(let i of n.items)o.push(B(i));for(let i of c(o))e.out(i);return e.out(`
+${n.items.length} trial(s) shown`),n.nextCursor&&e.out(`More: evolve-evals trials ${t.positionals[0]} --cursor ${n.nextCursor}`),0}async function nt(t,e){let s=await H$1(f(t)).trial(t.positionals[0],t.positionals[1]);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let n of G(s))e.out(n);return 0}async function it(t,e){let r=H$1(f(t)),s=t.flags.json===true,n=0;for await(let o of r.trialTraceEvents(t.positionals[0],t.positionals[1],{...t.flags.cursor!==void 0?{cursor:String(t.flags.cursor)}:{},...t.flags.limit!==void 0?{limit:t.flags.limit}:{}}))e.out(s?JSON.stringify(o):z(o)),n+=1;return !s&&n===0&&e.out("No trace events."),0}function ot(t){let e=[],r=[["ID","BENCHMARK","STATUS","MEAN REWARD","COVERAGE","SPENT"]];for(let s of t.jobs)r.push([s.id,s.benchmark,s.status,k(s.meanReward),`${s.coverage.scored}/${s.coverage.total}`,h(s.spentUsd)]);if(e.push(...c(r)),t.taskMatrix.length>0){e.push("","Task matrix (disagreements first; columns in the order above):");let s=[["TASK","DIFF",...t.jobs.map((o,i)=>`JOB ${i+1}`)]],n=t.jobs.map(o=>o.id);for(let o of t.taskMatrix){let i=new Map(o.cells.map(a=>[a.jobId,a]));s.push([o.taskKey,o.disagreement?"!":"",...n.map(a=>{let u=i.get(a);return u?u.meanReward!==null?`${u.status} ${k(u.meanReward)}`:u.status:"-"})]);}e.push(...c(s));}return e}async function at(t,e){let s=await H$1(f(t)).compare(t.positionals);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let n of ot(s))e.out(n);return 0}async function ut(t,e){let s=await H$1(f(t)).cancel(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(s));else for(let n of b(s))e.out(n);return 0}async function lt(t,e){let s=await H$1(f(t)).rerunFailed(t.positionals[0]);if(t.flags.json===true)e.out(JSON.stringify(s));else {for(let n of b(s))e.out(n);e.out(""),e.out(`Follow it with: evolve-evals get ${s.id}`);}return 0}async function ct(t,e){let r=H$1(f(t)),[s,n]=t.positionals,o;if(n!==void 0){if(t.flags.status!==void 0||t.flags.task!==void 0)throw new l("--status/--task apply to a whole-job regrade, not a single trial");o=await r.regradeTrial(s,n);}else {let i={};if(t.flags.status!==void 0){let a=String(t.flags.status).split(",").map(u=>u.trim()).filter(Boolean);if(a.length===0)throw new l("--status got an empty status list");i.status=a;}t.flags.task!==void 0&&(i.taskKey=String(t.flags.task)),o=await r.regrade(s,i);}if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of P(o))e.out(i);e.out(""),e.out(`Follow it with: evolve-evals regrade-job ${o.id}`);}return 0}async function ft(t,e){let s=await H$1(f(t)).getRegrade(t.positionals[0],y(t));if(t.flags.json===true)e.out(JSON.stringify(s));else for(let n of P(s))e.out(n);return 0}async function dt(t,e){let r=t.flags.format;if(r!==void 0&&r!=="harbor")throw new l(`Unknown --format "${r}" (supported: harbor)`);let n=await H$1(f(t)).export(t.positionals[0],{to:t.flags.to??process.cwd(),...r==="harbor"?{format:"harbor"}:{}});return t.flags.json===true?e.out(JSON.stringify({path:n})):e.out(`Saved ${n}`),0}function mt(t){let e=c([["name",t.name],["title",t.title??"-"],["description",t.description??"-"],["active version",t.activeVersion?.version??"-"]]);if(t.versions&&t.versions.length>0){e.push("");let r=[["VERSION","STATE","TASKS","CREATED"]];for(let s of t.versions)r.push([s.version,s.state,String(s.taskCount),s.createdAt??"-"]);e.push(...c(r));}if(t.tasks&&t.tasks.items.length>0){e.push("",`Tasks (version ${t.selectedVersion?.version??"?"}):`);let r=[["TASK","AGENT TIMEOUT","VERIFIER TIMEOUT","PROVIDERS"]];for(let n of t.tasks.items)r.push([n.taskKey,`${n.agentTimeoutSec}s`,`${n.verifierTimeoutSec}s`,Y(n.providers)]);e.push(...c(r)),t.tasks.nextCursor&&e.push(`More tasks: evolve-evals benchmarks get ${t.name} --cursor ${t.tasks.nextCursor}`);let s=new Map;for(let n of t.tasks.items)for(let o of C){let i=n.providers?.[o];i&&!i.ok&&!s.has(`${o}:${i.reason}`)&&s.set(`${o}:${i.reason}`,`${o}: ${i.reason}`);}if(s.size>0){e.push("","Provider limitations:");for(let n of s.values())e.push(`  ${n}`);}}return e}async function gt(t,e){let r=F$1(f(t)),[s,n]=t.positionals;if(s===void 0){let i=await r.list(y(t));if(t.flags.json===true)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No benchmarks."),0;let a=[["NAME","ACTIVE","STATE","TASKS","TITLE"]];for(let u of i.items)a.push([u.name,u.activeVersion?.version??"-",u.activeVersion?.state??"-",u.activeVersion?String(u.activeVersion.taskCount):"-",u.title??"-"]);for(let u of c(a))e.out(u);for(let u of x(i.items))e.out(u);return 0}if(s!=="get")throw new l(`Unknown benchmarks subcommand "${s}" (did you mean "benchmarks get ${s}"?)`);if(!n)throw new l('"benchmarks get" requires a <name[@version]> ref');let o=await r.get(n,y(t));if(t.flags.json===true)e.out(JSON.stringify(o));else {for(let i of mt(o))e.out(i);for(let i of x([o]))e.out(i);}return 0}function x(t){let e=[];for(let r of t){if(!r.upstream?.moved)continue;let s=r.activeVersion?`@${r.activeVersion.version}`:"";e.push(`${r.name}${s} \xB7 upstream ${r.upstream.ref} moved \u2014 run: evolve-evals import --benchmark ${r.name} --version <new-version> --git-url <url> --ref ${r.upstream.ref}`);}return e}async function pt(t,e){let[r,s]=t.positionals,n=t.flags.json===true,o=F$1(f(t));if(r==="status"){if(!s)throw new l('"import status" requires an <id>');let d=await o.getImport(s);if(n)e.out(JSON.stringify(d));else for(let w of I(d))e.out(w);return 0}if(r!==void 0)throw new l(`Unknown import subcommand "${r}" (supported: status)`);let i=M(t),a=await o.import(i);if(t.flags.watch!==true){if(n)e.out(JSON.stringify(a));else {for(let d of I(a))e.out(d);e.out(""),e.out(`Follow it with: evolve-evals import status ${a.id}`);}return 0}n?e.out(JSON.stringify({kind:"import.created",benchmarkImport:a})):e.out(`Import ${a.id} (${i.benchmarkName}) ${a.status} \u2014 watching\u2026`);let u=await o.watchImport(a.id,{onStatus:d=>{e.out(n?JSON.stringify({kind:"import.status",benchmarkImport:d}):Q(d));}});if(n)e.out(JSON.stringify({kind:"import.final",benchmarkImport:u}));else {e.out("");for(let d of I(u))e.out(d);}return u.status==="FAILED"?1:0}async function ht(t,e){let r=G$1(f(t)),[s,n]=t.positionals,o=t.flags.json===true;if(s===void 0){let i=await r.list(y(t));if(o)return e.out(JSON.stringify(i)),0;if(i.items.length===0)return e.out("No custom harnesses."),0;let a=[["NAME","SOURCE","RUN COMMAND","UPDATED"]];for(let u of i.items)a.push([u.name,u.source,v(u.runCommand,60),u.updatedAt]);for(let u of c(a))e.out(u);return 0}if(s==="add"){if(n!==void 0)throw new l(`"custom-harnesses add" got unexpected argument "${n}"`);let i=await r.create(V(t));if(o)e.out(JSON.stringify(i));else {for(let a of R(i))e.out(a);e.out(""),e.out(`Use it with: evolve-evals run --agent ${i.name}:<model> \u2026`);}return 0}if(s==="get"){if(!n)throw new l('"custom-harnesses get" requires a <name>');let i=await r.get(n);if(o)e.out(JSON.stringify(i));else for(let a of R(i))e.out(a);return 0}if(s==="remove"){if(!n)throw new l('"custom-harnesses remove" requires a <name>');return await r.delete(n),o?e.out(JSON.stringify({name:n,deleted:true})):e.out(`Deleted custom harness ${n}`),0}throw new l(`Unknown custom-harnesses subcommand "${s}" (supported: add, get, remove)`)}async function wt(t,e=F){let r;try{r=L(t);}catch(s){if(s instanceof l)return e.err(`Error: ${s.message}`),e.err('Run "evolve-evals help" for usage.'),2;throw s}try{switch(r.command){case "help":return e.out(j),0;case "run":return await tt(r,e);case "list":return await et(r,e);case "get":return await st(r,e);case "trials":return await rt(r,e);case "trial":return await nt(r,e);case "trace":return await it(r,e);case "compare":return await at(r,e);case "cancel":return await ut(r,e);case "rerun-failed":return await lt(r,e);case "regrade":return await ct(r,e);case "regrade-job":return await ft(r,e);case "export":return await dt(r,e);case "benchmarks":return await gt(r,e);case "import":return await pt(r,e);case "custom-harnesses":return await ht(r,e);default:return e.err(`Error: unknown command "${r.command}"`),2}}catch(s){return s instanceof l?(e.err(`Error: ${s.message}`),e.err('Run "evolve-evals help" for usage.'),2):(e.err(`Error: ${s.message}`),1)}}var bt=(()=>{let t=process.argv[1];if(!t)return  false;try{return import.meta.url===pathToFileURL(t).href}catch{return  false}})();bt&&wt(process.argv.slice(2)).then(t=>{process.exitCode=t;},t=>{process.stderr.write(`Error: ${t?.message??t}
+`),process.exitCode=1;});export{l as CliUsageError,j as USAGE,V as buildCustomHarnessInput,M as buildImportInput,q as buildJobInput,X as eventLine,Q as importStatusLine,L as parseArgs,K as parseHarnessEnv,D as parseJobAgent,wt as runCli,z as traceEventLine};