@rulemetric/cli 0.7.27 → 0.7.29

package/dist/{chunk-WXFFD4IV.js → chunk-4O7MJUPH.js}

@@ -1 +1 @@
-import{a as $}from"./chunk-CHA5SX4Y.js";import{e as w,f as k}from"./chunk-H2ORWIEY.js";import{a as u}from"./chunk-ZLLER3HP.js";function h(e){if(e.length===0)return{mean:0,stddev:0,min:0,max:0};let n=e.reduce((s,a)=>s+a,0)/e.length,o=e.reduce((s,a)=>s+(a-n)**2,0)/e.length;return{mean:Math.round(n*1e3)/1e3,stddev:Math.round(Math.sqrt(o)*1e3)/1e3,min:Math.min(...e),max:Math.max(...e)}}u(h,"computeStats");var x="with_target",E="without_target";function _(e){let n=e.grading.summary?.pass_rate;return typeof n=="number"&&n>=1}u(_,"fullPass");function b(e){return e.grading.summary?.pass_rate!=null}u(b,"hasGrading");function R(e){let n=e.timing.total_tokens;return typeof n=="number"&&n>0?n:void 0}u(R,"tokensOf");function v(e){let n=e.timing.duration_ms;return typeof n=="number"&&!isNaN(n)?n/1e3:void 0}u(v,"secondsOf");function y(e,n){let o={};for(let s of e)(o[n(s)]??=[]).push(s);return o}u(y,"groupBy");function C(e,n){let o=y(e.filter(b),i=>i.evalId),s=y(n.filter(b),i=>i.evalId),a=u((i,l)=>i.runNumber-l.runNumber,"byRun"),t=[];for(let i of Object.keys(o)){let l=o[i].sort(a),d=(s[i]??[]).sort(a);for(let r=0;r<Math.min(l.length,d.length);r++){let c={withScore:_(l[r])?1:0,withoutScore:_(d[r])?1:0},f=R(l[r]),g=R(d[r]);f!=null&&g!=null&&(c.withCost=f,c.withoutCost=g);let p=v(l[r]),m=v(d[r]);p!=null&&m!=null&&(c.withLatency=p,c.withoutLatency=m),t.push(c)}}return t}u(C,"pairRunsByEval");function T(e){let n=e.filter(r=>r.status==="completed");if(n.length===0)return{runSummary:{},verdict:null,notes:["No completed runs found"],totalRuns:0};let o=y(n,r=>r.configuration),s={},a=[];for(let[r,c]of Object.entries(o)){let f=c.filter(b).map(m=>m.grading.summary.pass_rate),g=c.map(v).filter(m=>m!=null),p=c.map(R).filter(m=>m!=null);s[r]={},f.length>0&&(s[r].pass_rate=h(f)),g.length>0&&(s[r].time_seconds=h(g)),p.length>0&&(s[r].tokens=h(p))}let t=null,i=o[x]??[],l=o[E]??[],d=C(i,l);if(d.length>0){if(t=$(d),!t.sufficient)a.push(t.insufficientReason??"Insufficient data for a significance verdict.");else if(t.significant){let r=t.delta>=0?"improves":"reduces",c=t.pValue!=null?`, McNemar p=${t.pValue.toFixed(4)}`:"";a.push(`Target ${r} the pass rate by ${Math.abs(t.delta*100).toFixed(0)}% (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}${c}).`)}else a.push(`No significant effect on pass rate (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}).`);t.costDelta!=null&&Math.round(t.costDelta)!==0&&a.push(`Token cost delta: ${t.costDelta>=0?"+":""}${Math.round(t.costDelta)} tokens/task.`),t.latencyDelta!=null&&Math.abs(t.latencyDelta)>1&&a.push(`Target ${t.latencyDelta>0?"adds":"saves"} ${Math.abs(t.latencyDelta).toFixed(1)}s/task.`)}for(let r of Object.values(s))if(r.pass_rate&&r.pass_rate.stddev>.2){a.push(`High pass-rate variance (stddev=${r.pass_rate.stddev.toFixed(2)}) \u2014 results may be unreliable.`);break}return{runSummary:s,verdict:t,notes:a,totalRuns:n.length}}u(T,"summarizeRuns");async function S(e,n){let o=n?`&iteration=${n}`:"",s=100,a=[];for(let t=0;t<1e5;t+=s){let i=await w(`/api/eval-targets/${e}/runs?limit=${s}&offset=${t}&activeCasesOnly=1${o}`);if(a.push(...i),i.length<s)break}return T(a)}u(S,"aggregateBenchmark");async function B(e,n,o){return k(`/api/eval-targets/${e}/benchmarks`,{iteration:n,runSummary:o.runSummary,notes:o.notes})}u(B,"postBenchmark");export{T as a,S as b,B as c};
+import{a as $}from"./chunk-X2ILN3KP.js";import{e as w,f as k}from"./chunk-H2ORWIEY.js";import{a as u}from"./chunk-ZLLER3HP.js";function h(e){if(e.length===0)return{mean:0,stddev:0,min:0,max:0};let n=e.reduce((s,a)=>s+a,0)/e.length,o=e.reduce((s,a)=>s+(a-n)**2,0)/e.length;return{mean:Math.round(n*1e3)/1e3,stddev:Math.round(Math.sqrt(o)*1e3)/1e3,min:Math.min(...e),max:Math.max(...e)}}u(h,"computeStats");var x="with_target",E="without_target";function _(e){let n=e.grading.summary?.pass_rate;return typeof n=="number"&&n>=1}u(_,"fullPass");function b(e){return e.grading.summary?.pass_rate!=null}u(b,"hasGrading");function R(e){let n=e.timing.total_tokens;return typeof n=="number"&&n>0?n:void 0}u(R,"tokensOf");function v(e){let n=e.timing.duration_ms;return typeof n=="number"&&!isNaN(n)?n/1e3:void 0}u(v,"secondsOf");function y(e,n){let o={};for(let s of e)(o[n(s)]??=[]).push(s);return o}u(y,"groupBy");function C(e,n){let o=y(e.filter(b),i=>i.evalId),s=y(n.filter(b),i=>i.evalId),a=u((i,l)=>i.runNumber-l.runNumber,"byRun"),t=[];for(let i of Object.keys(o)){let l=o[i].sort(a),d=(s[i]??[]).sort(a);for(let r=0;r<Math.min(l.length,d.length);r++){let c={withScore:_(l[r])?1:0,withoutScore:_(d[r])?1:0},f=R(l[r]),g=R(d[r]);f!=null&&g!=null&&(c.withCost=f,c.withoutCost=g);let p=v(l[r]),m=v(d[r]);p!=null&&m!=null&&(c.withLatency=p,c.withoutLatency=m),t.push(c)}}return t}u(C,"pairRunsByEval");function T(e){let n=e.filter(r=>r.status==="completed");if(n.length===0)return{runSummary:{},verdict:null,notes:["No completed runs found"],totalRuns:0};let o=y(n,r=>r.configuration),s={},a=[];for(let[r,c]of Object.entries(o)){let f=c.filter(b).map(m=>m.grading.summary.pass_rate),g=c.map(v).filter(m=>m!=null),p=c.map(R).filter(m=>m!=null);s[r]={},f.length>0&&(s[r].pass_rate=h(f)),g.length>0&&(s[r].time_seconds=h(g)),p.length>0&&(s[r].tokens=h(p))}let t=null,i=o[x]??[],l=o[E]??[],d=C(i,l);if(d.length>0){if(t=$(d),!t.sufficient)a.push(t.insufficientReason??"Insufficient data for a significance verdict.");else if(t.significant){let r=t.delta>=0?"improves":"reduces",c=t.pValue!=null?`, McNemar p=${t.pValue.toFixed(4)}`:"";a.push(`Target ${r} the pass rate by ${Math.abs(t.delta*100).toFixed(0)}% (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}${c}).`)}else a.push(`No significant effect on pass rate (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}).`);t.costDelta!=null&&Math.round(t.costDelta)!==0&&a.push(`Token cost delta: ${t.costDelta>=0?"+":""}${Math.round(t.costDelta)} tokens/task.`),t.latencyDelta!=null&&Math.abs(t.latencyDelta)>1&&a.push(`Target ${t.latencyDelta>0?"adds":"saves"} ${Math.abs(t.latencyDelta).toFixed(1)}s/task.`)}for(let r of Object.values(s))if(r.pass_rate&&r.pass_rate.stddev>.2){a.push(`High pass-rate variance (stddev=${r.pass_rate.stddev.toFixed(2)}) \u2014 results may be unreliable.`);break}return{runSummary:s,verdict:t,notes:a,totalRuns:n.length}}u(T,"summarizeRuns");async function S(e,n){let o=n?`&iteration=${n}`:"",s=100,a=[];for(let t=0;t<1e5;t+=s){let i=await w(`/api/eval-targets/${e}/runs?limit=${s}&offset=${t}&activeCasesOnly=1${o}`);if(a.push(...i),i.length<s)break}return T(a)}u(S,"aggregateBenchmark");async function B(e,n,o){return k(`/api/eval-targets/${e}/benchmarks`,{iteration:n,runSummary:o.runSummary,notes:o.notes})}u(B,"postBenchmark");export{T as a,S as b,B as c};

package/dist/{chunk-GQRCSR2A.js → chunk-4VZLIGMS.js}

@@ -1,4 +1,4 @@
-import{b as p}from"./chunk-WQ2BJQYB.js";import{a as c}from"./chunk-ZLLER3HP.js";var y=`You are an expert at improving AI coding instructions, skills, and prompts. You receive eval results showing how well the current content performs, and your job is to propose specific improvements.
+import{c as p}from"./chunk-HMMO27U4.js";import{a as c}from"./chunk-ZLLER3HP.js";var y=`You are an expert at improving AI coding instructions, skills, and prompts. You receive eval results showing how well the current content performs, and your job is to propose specific improvements.
 
 You will receive:
 1. The current content of the eval target

package/dist/chunk-6N6NTMOR.js

@@ -0,0 +1 @@
+import{c as P}from"./chunk-TIOKY2IA.js";import{a as T}from"./chunk-M4YF7YO7.js";import{a as A}from"./chunk-RSKQRVPJ.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as M}from"./chunk-6UTJGOBA.js";import{a as S}from"./chunk-OSIP6FUD.js";import{a as h}from"./chunk-ULRVMH7O.js";import{a as E}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-T4P2NV3P.js";import{b as I}from"./chunk-BUWWFHGO.js";import{b}from"./chunk-5RBLIVBF.js";import{a as H}from"./chunk-CS2OQ3GC.js";import{a as N}from"./chunk-DUP232AH.js";import{a as x}from"./chunk-DO3KHK2X.js";import{a as k}from"./chunk-UF6US7VU.js";import{a as v}from"./chunk-MSTF7A7D.js";import{b as R}from"./chunk-4HZWSFSY.js";import{d as w}from"./chunk-3BVHPQZW.js";import{b as f,e as _,f as p}from"./chunk-H2ORWIEY.js";import{a as s}from"./chunk-ZLLER3HP.js";var O={process_eval:h,process_insights:E,process_launch:I,process_send_message:A,process_conversation:S,process_session_goal:y,distill_memories:R,memory_ab:H,process_announcement:M,process_run_cleanup:T,cron_suggest_instructions:v,cron_refresh_skills:k,cron_refresh_pricing:x,cron_eval_autorun:N,process_judge_review:L},C=Object.keys(O),$=new Set(["process_session_goal","process_announcement","process_run_cleanup","distill_memories","memory_ab"]);function F(e=process.env){return e.DATABASE_URL?C:C.filter(n=>!$.has(n))}s(F,"availableCapabilities");var J=3e4,U=3e4,K=3e4,D=250,V=5e3,G=`thin-agent-${process.pid}`,g=null;function q(e,n,r,l){let c;try{let d=w();c={claude:d.claudeBin!=null,codex:d.codexBin!=null,codexModel:d.codexConfiguredModel}}catch{}let i=s(d=>{p("/api/workers/heartbeat",{workerType:"cli-worker",clientName:G,clientVersion:l,workerVersion:"0.1.0",currentJobId:null,inFlightCount:0,lastError:null,status:d,metadata:{capabilities:e,mode:"thin-agent",llmEngines:c}}).then(u=>{let t=u?.advisory;t&&t.message!==g?(g=t.message,a(r,{event:"worker_advisory",code:t.code,message:t.message})):t||(g=null)}).catch(u=>{a(r,{event:"presence_heartbeat_error",error:u instanceof Error?u.message:String(u)})})},"send");i("alive");let o=setInterval(()=>i("alive"),K);n.addEventListener("abort",()=>{clearInterval(o),i("shutting_down")},{once:!0})}s(q,"startPresenceHeartbeat");function a(e,n){try{e(JSON.stringify({ts:new Date().toISOString(),...n}))}catch{e(`[agent-loop] log_error event=${n.event}`)}}s(a,"structuredLog");function B(e,n){return n.aborted?Promise.resolve(!1):e<=0?Promise.resolve(!0):new Promise(r=>{let l=setTimeout(()=>{n.removeEventListener("abort",c),r(!0)},e),c=s(()=>{clearTimeout(l),n.removeEventListener("abort",c),r(!1)},"onAbort");n.addEventListener("abort",c,{once:!0})})}s(B,"abortableSleep");async function Q(e,n){let r=`capabilities=${encodeURIComponent(e.join(","))}&waitMs=${n}`,l=await _(`/api/work/next?${r}`);return l==null||typeof l=="string"?null:l}s(Q,"pollForWork");function X(e,n){return{logger:{info:s(r=>n(r),"info"),warn:s(r=>n(r),"warn"),error:s(r=>n(r),"error")},job:{attempts:e}}}s(X,"makeHelpers");async function z(e,n,r){let l=O[e.kind],c=Date.now();if(a(r,{event:"claim",jobId:e.id,kind:e.kind,attempts:e.attempts}),!l){let o=`unknown kind: ${e.kind}`;a(r,{event:"fail",jobId:e.id,kind:e.kind,terminal:!0,error:o}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:o,terminal:!0}).catch(()=>{});return}let i=setInterval(()=>{a(r,{event:"heartbeat",jobId:e.id}),p(`/api/work/${e.id}/heartbeat`,{leaseToken:e.leaseToken}).catch(o=>{a(r,{event:"heartbeat_error",jobId:e.id,error:o instanceof Error?o.message:String(o)})})},n);try{let o=X(e.attempts,r),d=await l(e.payload,o)??{};clearInterval(i),a(r,{event:"complete",jobId:e.id,kind:e.kind,durationMs:Date.now()-c}),await p(`/api/work/${e.id}/complete`,{leaseToken:e.leaseToken,result:d})}catch(o){clearInterval(i);let d=b(o),u=o instanceof Error?o.message:String(o);a(r,{event:"fail",jobId:e.id,kind:e.kind,durationMs:Date.now()-c,terminal:d,error:u}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:u,terminal:d}).catch(t=>{a(r,{event:"fail_post_error",jobId:e.id,error:t instanceof Error?t.message:String(t)})})}}s(z,"runOne");async function ve(e){let n=e.log??(()=>{}),r=e.pollWaitMs??J,l=e.heartbeatIntervalMs??U,c=e.capabilities??F(),i=D;a(n,{event:"agent_start",capabilities:c,pollWaitMs:r,heartbeatIntervalMs:l}),q(c,e.signal,n,e.version),P({signal:e.signal,log:n}).catch(t=>{a(n,{event:"tt_heartbeat_fatal",error:t instanceof Error?t.message:String(t)})});let o=5*6e4,d=0,u=s(()=>{if(!e.readInstalledVersion||!e.version)return!1;let t=Date.now();if(t-d<o)return!1;d=t;let m=e.readInstalledVersion();return!!(m&&m!==e.version)},"versionDrifted");for(;!e.signal.aborted;){if(u()){a(n,{event:"agent_version_drift",bootVersion:e.version,installedVersion:e.readInstalledVersion?.()??null}),e.onVersionDrift?.();break}let t=null;try{t=await Q(c,r)}catch(m){let W=m instanceof f?m.statusCode:void 0;if(a(n,{event:"poll_error",status:W,error:m instanceof Error?m.message:String(m)}),!await B(i,e.signal))break;i=Math.min(i*2,V);continue}if(!t){if(a(n,{event:"poll_204"}),!await B(i,e.signal))break;i=Math.min(i*2,V);continue}i=D,await z(t,l,n)}a(n,{event:"agent_stop"})}s(ve,"runAgentLoop");export{O as a,C as b,F as c,Q as d,ve as e};

package/dist/{chunk-LVOLDUFC.js → chunk-6UTJGOBA.js}

@@ -1,2 +1,2 @@
-import{a as R}from"./chunk-5RBLIVBF.js";import{b as B}from"./chunk-N7TGRFVZ.js";import{a as v,g as d,h as n,i as m,j as U}from"./chunk-KZ65LEUH.js";import{a as $}from"./chunk-ZLLER3HP.js";import{and as L,asc as j,eq as l,gt as q,isNotNull as F,sql as O}from"drizzle-orm";var _=100,C=10;async function G(I,s={}){let t=s.db??U(),D=s.sendEmail??B,p=s.log??(o=>process.stderr.write(`[process_announcement] ${o}
+import{b as B}from"./chunk-N7TGRFVZ.js";import{a as R}from"./chunk-5RBLIVBF.js";import{a as v,g as d,h as n,i as m,j as U}from"./chunk-KZ65LEUH.js";import{a as $}from"./chunk-ZLLER3HP.js";import{and as L,asc as j,eq as l,gt as q,isNotNull as F,sql as O}from"drizzle-orm";var _=100,C=10;async function G(I,s={}){let t=s.db??U(),D=s.sendEmail??B,p=s.log??(o=>process.stderr.write(`[process_announcement] ${o}
 `)),b=s.webBaseUrl??process.env.RULEMETRIC_WEB_URL??process.env.WEB_URL??"https://rulemetric.com",{announcementId:r}=I,[i]=await t.select().from(d).where(l(d.id,r)).limit(1);if(!i)throw new R(`Announcement ${r} not found`);if(i.status!=="queued"&&i.status!=="sending")throw new R(`Announcement ${r} is in status='${i.status}'; expected 'queued' or 'sending'.`);await t.update(d).set({status:"sending",updatedAt:new Date}).where(l(d.id,r)),p(`Sending announcement ${r} (${i.subject??""})`);let w=0,f=0,h=0,y=0,k=null;try{for(;;){let c=await t.select({userId:m.userId,email:v.email,displayName:v.displayName,unsubscribeToken:m.unsubscribeToken}).from(m).innerJoin(v,l(v.id,m.userId)).where(L(l(m.productAnnouncements,!0),F(v.email),k?q(m.userId,k):void 0)).orderBy(j(m.userId)).limit(_);if(c.length===0)break;k=c[c.length-1].userId;for(let A=0;A<c.length;A+=C){let P=c.slice(A,A+C),S=await Promise.allSettled(P.map(g=>W({recipient:g,announcement:i,webBaseUrl:b,sendEmail:D,db:t})));for(let g=0;g<S.length;g++){let u=S[g];if(u.status==="fulfilled")u.value.ok==="sent"?w++:u.value.ok==="skipped"?h++:(f++,u.value.provider==="noop"&&y++,p(`  recipient ${P[g].userId}: send failed \u2014 ${u.value.error}`));else{f++;let T=u.reason instanceof Error?u.reason.message:String(u.reason);p(`  recipient ${P[g].userId}: send failed \u2014 ${T}`)}}}if(c.length<_)break}y>0&&p(`WARNING: ${y} recipient(s) hit the noop email provider and were recorded as FAILED \u2014 no email provider is configured on this worker (set RESEND_API_KEY in ~/.config/rulemetric/env and reinstall the service)`);let o=await t.select({status:n.status,count:O`count(*)::int`}).from(n).where(l(n.announcementId,r)).groupBy(n.status),e={};for(let c of o)e[c.status]=c.count;let a=e.sent??0,E=e.failed??0,N=a===0&&E>0?"failed":"sent",x=E>0?`${E} of ${a+E} recipient(s) failed; see announcement_deliveries`:null;return await t.update(d).set({status:N,recipientCount:a,...N==="sent"?{sentAt:new Date}:{},error:x,updatedAt:new Date}).where(l(d.id,r)),p(`Done: status=${N} sent=${a} failed=${E} (this attempt: sent=${w} failed=${f} skipped=${h})`),{sent:w,failed:f,skipped:h}}catch(o){let e=o instanceof Error?o.message:String(o);throw await t.update(d).set({status:"failed",error:e.slice(0,1e3),updatedAt:new Date}).where(l(d.id,r)).catch(()=>{}),o}}$(G,"processAnnouncement");function K(I){return I.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#39;")}$(K,"escapeHtml");async function W(I){let{recipient:s,announcement:t,webBaseUrl:D,sendEmail:p,db:b}=I;if(!s.email)return{ok:"skipped",reason:"no-email"};let r=await b.insert(n).values({announcementId:t.id,userId:s.userId,email:s.email,status:"pending"}).onConflictDoNothing({target:[n.announcementId,n.userId]}).returning({id:n.id});if(r.length===0)return{ok:"skipped",reason:"already-claimed"};let i=r[0].id,w=D.replace(/\/$/,""),f=encodeURIComponent(s.unsubscribeToken),h=`${w}/unsubscribe/${f}`,y=s.displayName?.trim().split(/\s+/)[0]||"there",k=t.htmlBody.replaceAll("{{unsubscribe_url}}",h).replaceAll("{{recipient_name}}",K(y)),o=t.textBody.replaceAll("{{unsubscribe_url}}",h).replaceAll("{{recipient_name}}",y);try{let e=await p({to:s.email,subject:t.subject,html:k,text:o,fromName:"RuleMetric",idempotencyKey:i,headers:{"List-Unsubscribe":`<${w}/api/unsubscribe/${f}>`,"List-Unsubscribe-Post":"List-Unsubscribe=One-Click"}});if(e.provider==="noop"){let a="no email provider configured (RESEND_API_KEY unset on the worker)";return await b.update(n).set({status:"failed",error:a}).where(l(n.id,i)),{ok:"failed",error:a,provider:e.provider}}return await b.update(n).set({status:"sent",providerId:e.id,sentAt:new Date}).where(l(n.id,i)),{ok:"sent",providerId:e.id,provider:e.provider}}catch(e){let a=e instanceof Error?e.message:String(e);return await b.update(n).set({status:"failed",error:a.slice(0,1e3)}).where(l(n.id,i)).catch(()=>{}),{ok:"failed",error:a}}}$(W,"sendOne");export{G as a};

package/dist/{chunk-W2MOTOQE.js → chunk-BUWWFHGO.js}

@@ -1 +1 @@
-import{b as l}from"./chunk-6QKI756T.js";import{a as p}from"./chunk-5RBLIVBF.js";import{h}from"./chunk-H2ORWIEY.js";import{a as c}from"./chunk-ZLLER3HP.js";async function g(t){if(!t.worktree?.create)return t;let{branch:e,baseRef:s}=t.worktree,n=await l({repoRoot:t.projectPath,branch:e,baseRef:s});return{...t,projectPath:n.path,gitBranch:n.branch}}c(g,"resolveWorktreeForLaunch");async function L(t,e){let{mirrorId:s,tool:n}=t,{config:r}=t,a=c(o=>e.logger.info(o),"log");if(e.job.attempts>1)throw new p(`process_launch is not safe to retry (attempt ${e.job.attempts}). Re-launch manually.`);r=await g(r),r.worktree?.create&&a(`Worktree resolved: ${r.projectPath} (branch ${r.gitBranch})`),a(`Launching ${n}...`);let{getLauncher:f}=await import("./dist-FAVALJ5C.js"),i=await f(n).launch(r);a(`Launch completed: sessionId=${i.sessionId}`);let u={...i,worktreePath:r.worktree?.create?r.projectPath:void 0,runId:r.runId};return await h(`/api/launch-jobs/${s}/state`,{result:u}).catch(o=>{a(`result persist failed: ${o instanceof Error?o.message:String(o)}`)}),u}c(L,"processLaunch");export{g as a,L as b};
+import{b as l}from"./chunk-MDPN46EY.js";import{a as p}from"./chunk-5RBLIVBF.js";import{h}from"./chunk-H2ORWIEY.js";import{a as c}from"./chunk-ZLLER3HP.js";async function g(t){if(!t.worktree?.create)return t;let{branch:e,baseRef:s}=t.worktree,n=await l({repoRoot:t.projectPath,branch:e,baseRef:s});return{...t,projectPath:n.path,gitBranch:n.branch}}c(g,"resolveWorktreeForLaunch");async function L(t,e){let{mirrorId:s,tool:n}=t,{config:r}=t,a=c(o=>e.logger.info(o),"log");if(e.job.attempts>1)throw new p(`process_launch is not safe to retry (attempt ${e.job.attempts}). Re-launch manually.`);r=await g(r),r.worktree?.create&&a(`Worktree resolved: ${r.projectPath} (branch ${r.gitBranch})`),a(`Launching ${n}...`);let{getLauncher:f}=await import("./dist-BC6GMBXV.js"),i=await f(n).launch(r);a(`Launch completed: sessionId=${i.sessionId}`);let u={...i,worktreePath:r.worktree?.create?r.projectPath:void 0,runId:r.runId};return await h(`/api/launch-jobs/${s}/state`,{result:u}).catch(o=>{a(`result persist failed: ${o instanceof Error?o.message:String(o)}`)}),u}c(L,"processLaunch");export{g as a,L as b};

package/dist/{chunk-2TX7TN5R.js → chunk-CBP7QZAN.js}

@@ -1 +1 @@
-import{c as m,d as p}from"./chunk-ZL4MZMUK.js";import{a as C}from"./chunk-N7TGRFVZ.js";import{g as c,j as f}from"./chunk-KZ65LEUH.js";import{a as r}from"./chunk-ZLLER3HP.js";import*as a from"node:fs/promises";import*as g from"node:path";import{eq as b}from"drizzle-orm";async function v(e){let n=e;for(let t=0;t<50;t++){try{return await a.stat(g.join(n,"pnpm-workspace.yaml")),n}catch{}let i=g.dirname(n);if(i===n)return null;n=i}return null}r(v,"findRepoRoot");async function k(e){if(e)try{return await a.readFile(e,"utf-8")}catch{return null}let n=await v(process.cwd())??process.cwd(),t=g.join(n,"CHANGELOG.md");try{return await a.readFile(t,"utf-8")}catch{return null}}r(k,"readChangelog");var _="https://rulemetric.com";function D(e){let n=e.date??new Date().toISOString().slice(0,10),t=C({version:e.version,date:n,sections:e.sections,unsubscribeUrl:"{{unsubscribe_url}}",recipientName:"{{recipient_name}}",productUrl:_});return{version:e.version,releaseDate:n,subject:t.subject,htmlBody:t.html,textBody:t.text,sections:e.sections}}r(D,"renderRow");async function R(e={}){let n=e.db??f(),t=e.logger??{info:r(s=>console.log(s),"info"),warn:r(s=>console.warn(s),"warn")},i=await k(e.changelogPath);if(i===null)return t.info("[cron_check_changelog] CHANGELOG.md not found; nothing to do"),{createdCount:0,skippedCount:0};let y=m(i),h=p(y);if(h.length===0)return t.info("[cron_check_changelog] No announceable entries; nothing to do"),{createdCount:0,skippedCount:0};let d=0,l=0;for(let s of h){if((await n.select({id:c.id}).from(c).where(b(c.version,s.version)).limit(1)).length>0){l++;continue}let o=D(s);try{await n.insert(c).values({version:o.version,releaseDate:o.releaseDate,subject:o.subject,htmlBody:o.htmlBody,textBody:o.textBody,sections:o.sections,status:"draft"}),d++,t.info(`[cron_check_changelog] Created draft announcement for ${o.version}`)}catch(u){let w=u instanceof Error?u.message:String(u);if(/duplicate key|unique/i.test(w)){l++;continue}throw u}}return t.info(`[cron_check_changelog] Done: created ${d}, skipped ${l}`),{createdCount:d,skippedCount:l}}r(R,"processChangelog");export{R as a};
+import{a as C}from"./chunk-N7TGRFVZ.js";import{c as m,d as p}from"./chunk-ZL4MZMUK.js";import{g as c,j as f}from"./chunk-KZ65LEUH.js";import{a as r}from"./chunk-ZLLER3HP.js";import*as a from"node:fs/promises";import*as g from"node:path";import{eq as b}from"drizzle-orm";async function v(e){let n=e;for(let t=0;t<50;t++){try{return await a.stat(g.join(n,"pnpm-workspace.yaml")),n}catch{}let i=g.dirname(n);if(i===n)return null;n=i}return null}r(v,"findRepoRoot");async function k(e){if(e)try{return await a.readFile(e,"utf-8")}catch{return null}let n=await v(process.cwd())??process.cwd(),t=g.join(n,"CHANGELOG.md");try{return await a.readFile(t,"utf-8")}catch{return null}}r(k,"readChangelog");var _="https://rulemetric.com";function D(e){let n=e.date??new Date().toISOString().slice(0,10),t=C({version:e.version,date:n,sections:e.sections,unsubscribeUrl:"{{unsubscribe_url}}",recipientName:"{{recipient_name}}",productUrl:_});return{version:e.version,releaseDate:n,subject:t.subject,htmlBody:t.html,textBody:t.text,sections:e.sections}}r(D,"renderRow");async function R(e={}){let n=e.db??f(),t=e.logger??{info:r(s=>console.log(s),"info"),warn:r(s=>console.warn(s),"warn")},i=await k(e.changelogPath);if(i===null)return t.info("[cron_check_changelog] CHANGELOG.md not found; nothing to do"),{createdCount:0,skippedCount:0};let y=m(i),h=p(y);if(h.length===0)return t.info("[cron_check_changelog] No announceable entries; nothing to do"),{createdCount:0,skippedCount:0};let d=0,l=0;for(let s of h){if((await n.select({id:c.id}).from(c).where(b(c.version,s.version)).limit(1)).length>0){l++;continue}let o=D(s);try{await n.insert(c).values({version:o.version,releaseDate:o.releaseDate,subject:o.subject,htmlBody:o.htmlBody,textBody:o.textBody,sections:o.sections,status:"draft"}),d++,t.info(`[cron_check_changelog] Created draft announcement for ${o.version}`)}catch(u){let w=u instanceof Error?u.message:String(u);if(/duplicate key|unique/i.test(w)){l++;continue}throw u}}return t.info(`[cron_check_changelog] Done: created ${d}, skipped ${l}`),{createdCount:d,skippedCount:l}}r(R,"processChangelog");export{R as a};

package/dist/chunk-CDY7TQFZ.js

@@ -0,0 +1 @@
+import{a as r}from"./chunk-ZLLER3HP.js";function o(e){let n=e.match(/^\(\?([ims]+)\)/);return n?{source:e.slice(n[0].length),flags:[...new Set(n[1])].join("")}:{source:e,flags:""}}r(o,"translateInlineFlags");var c=[{prefix:"contains:",kind:"contains"},{prefix:"icontains:",kind:"icontains"},{prefix:"not-contains:",kind:"not-contains"},{prefix:"regex:",kind:"regex"}];function d(e){if(e.trim()==="json-valid")return{kind:"json-valid",arg:""};for(let{prefix:n,kind:i}of c)if(e.startsWith(n))return{kind:i,arg:e.slice(n.length).trim()};return null}r(d,"parseDeterministicExpectation");function u(e,n){switch(e.kind){case"contains":return n.includes(e.arg)?{passed:!0,evidence:`Output contains "${e.arg}"`}:{passed:!1,evidence:`"${e.arg}" not found in output`};case"icontains":return n.toLowerCase().includes(e.arg.toLowerCase())?{passed:!0,evidence:`Output contains "${e.arg}" (case-insensitive)`}:{passed:!1,evidence:`"${e.arg}" not found in output (case-insensitive)`};case"not-contains":return n.includes(e.arg)?{passed:!1,evidence:`Forbidden substring "${e.arg}" is present in output`}:{passed:!0,evidence:`"${e.arg}" is absent from output`};case"regex":{let i;try{let{source:t,flags:a}=o(e.arg);i=new RegExp(t,a)}catch(t){return{passed:!1,errored:!0,evidence:`Invalid regex "${e.arg}": ${t instanceof Error?t.message:String(t)}`}}let s=n.match(i);return s?{passed:!0,evidence:`Pattern matched: "${s[0]}"`}:{passed:!1,evidence:`Pattern /${e.arg}/ did not match output`}}case"json-valid":{let i=[n.trim()],s=n.match(/```(?:json)?\s*([\s\S]*?)```/);s&&i.push(s[1].trim());for(let t of i)try{return JSON.parse(t),{passed:!0,evidence:"Output parses as valid JSON"}}catch{}return{passed:!1,evidence:"No parseable JSON found in output (checked whole output and fenced blocks)"}}}}r(u,"runDeterministicCheck");export{d as a,u as b};

package/dist/chunk-CS2OQ3GC.js

@@ -0,0 +1 @@
+import{e as u,f as t,j as x}from"./chunk-KZ65LEUH.js";import{a as E}from"./chunk-KAWNGQS2.js";import{a as A}from"./chunk-X2ILN3KP.js";import{c as P}from"./chunk-HMMO27U4.js";import{a as f}from"./chunk-ZLLER3HP.js";import{eq as s}from"drizzle-orm";var v=f(m=>(m.summary.pass_rate??0)>=1?1:0,"fullPassScore");async function q(m,d={}){let r=d.db??x(),w=d.executeEval??P,y=d.grade??E,n=d.now??Date.now,[e]=await r.select().from(t).where(s(t.id,m.experimentId)).limit(1);if(!e)return{skipped:"not_found"};if(e.status==="completed"||e.status==="failed")return{skipped:"already_terminal",status:e.status};let[g]=await r.select({body:u.body}).from(u).where(s(u.id,e.memoryId)).limit(1);if(!g)return await r.update(t).set({status:"failed",error:"memory no longer exists",completedAt:new Date(n())}).where(s(t.id,e.id)),{skipped:"memory_gone",status:"failed"};await r.update(t).set({status:"running",startedAt:new Date(n())}).where(s(t.id,e.id));let b=e.expectations??[],l=e.model??void 0,a=[],h=0;try{for(let i=0;i<e.trials;i++){let c=await w({prompt:e.taskPrompt,targetContent:g.body,model:l}),p=await w({prompt:e.taskPrompt,model:l});if(c.error||p.error){h++;continue}let[D,M]=await Promise.all([y({prompt:e.taskPrompt,output:c.output,expectations:b,model:l}),y({prompt:e.taskPrompt,output:p.output,expectations:b,model:l})]);a.push({withScore:v(D),withoutScore:v(M),withLatency:c.durationMs/1e3,withoutLatency:p.durationMs/1e3})}if(a.length===0){let i=`no trials completed (${h}/${e.trials} runs errored \u2014 is the claude CLI available?)`;return await r.update(t).set({status:"failed",error:i,completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"failed",requestedTrials:e.trials,completedTrials:0}}let o=A(a);return await r.update(t).set({status:"completed",result:{verdict:o,pairs:a,completedTrials:a.length,model:l??null},completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"completed",requestedTrials:e.trials,completedTrials:a.length,significant:o.significant,delta:o.delta}}catch(o){let i=o instanceof Error?o.message:String(o);return await r.update(t).set({status:"failed",error:i.slice(0,500),completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"failed",requestedTrials:e.trials,completedTrials:a.length}}}f(q,"processMemoryAb");export{q as a};

package/dist/chunk-DUP232AH.js

@@ -0,0 +1 @@
+import{a as d}from"./chunk-CJRYYV76.js";import{b as u}from"./chunk-ZW5TAV3L.js";import{e as o}from"./chunk-H2ORWIEY.js";import{a as c}from"./chunk-ZLLER3HP.js";async function w(e,g){let i=c(a=>g?.logger?.info?.(a)??console.log(a),"log"),{evalTargetId:n}=e,s=await o(`/api/eval-targets/${n}/judge-calibration`),t=d({powered:s.powerCheck.powered,mde:s.powerCheck.mde??Number.POSITIVE_INFINITY},s.caseDiscrimination??{total:0,discriminating:0,nonDiscriminating:0,unpaired:0},{runsCompleted:0,maxRuns:e.maxRuns??24,batchesRun:0,maxBatches:1});if(t.action==="stop")return i(`[cron_eval_autorun] ${n}: ${t.reason}`),{skipped:t.code,reason:t.reason};let r=await o(`/api/eval-targets/${n}`),l=(await o(`/api/eval-targets/${n}/evals`)).filter(a=>!a.retiredAt);if(l.length===0)return{skipped:"no-live-cases"};let{completed:m,failed:p}=await u({cases:l,targetContent:r.content,concurrency:2,runNumber:1,opts:{targetId:n,targetVersion:r.version??1,iteration:1,timeout:e.timeout??24e4,model:e.model,graderModel:e.graderModel,skipGrading:!1,targetType:r.type,targetName:r.name},onLog:i});return i(`[cron_eval_autorun] ${r.name}: ${m} completed, ${p} failed`),{ranRuns:m,reason:t.reason}}c(w,"cronEvalAutorun");export{w as a};

package/dist/{chunk-WDGJHNOW.js → chunk-H7VP4SDO.js}

@@ -1,4 +1,4 @@
-import{b as p}from"./chunk-WQ2BJQYB.js";import{a as l}from"./chunk-ZLLER3HP.js";var k=[{name:"git-commit-helper",description:"Writes conventional commit messages and PR descriptions from a diff"},{name:"sql-query-optimizer",description:"Analyzes slow SQL queries and proposes indexes and rewrites"},{name:"i18n-string-extractor",description:"Finds hardcoded UI strings and extracts them into translation files"}],T=`You are an AI coding assistant with access to a set of optional skills. When a user request matches a skill's description, you invoke that skill before answering; when no skill applies, you answer directly without one. Descriptions are your ONLY information about each skill.
+import{c as p}from"./chunk-HMMO27U4.js";import{a as l}from"./chunk-ZLLER3HP.js";var k=[{name:"git-commit-helper",description:"Writes conventional commit messages and PR descriptions from a diff"},{name:"sql-query-optimizer",description:"Analyzes slow SQL queries and proposes indexes and rewrites"},{name:"i18n-string-extractor",description:"Finds hardcoded UI strings and extracts them into translation files"}],T=`You are an AI coding assistant with access to a set of optional skills. When a user request matches a skill's description, you invoke that skill before answering; when no skill applies, you answer directly without one. Descriptions are your ONLY information about each skill.
 
 {{CATALOG}}
 

package/dist/{chunk-BSXUH5FD.js → chunk-HL755A5U.js}

@@ -1 +1 @@
-import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-YDRZDBHG.js";import{c as J}from"./chunk-K25S2MGU.js";import{b as B,c as R,d as N,e as z}from"./chunk-3BVHPQZW.js";import{e as f,g as T}from"./chunk-H2ORWIEY.js";import{a as p}from"./chunk-ZLLER3HP.js";function W(r,a,s){return s?r.reviewerModel===s||r.reviewerModel===`${a}:${s}`:r.reviewerEngine===a}p(W,"reviewMatchesReviewer");var C=100;function Z(r){let a=new Set;return r.forEach((s,d)=>{s.method!=="deterministic"&&a.add(d)}),a}p(Z,"llmExpectationIndexes");var w=class extends Error{constructor(s,d){super(d);this.reason=s}static{p(this,"JudgeReviewSkip")}};async function oe(r){let a=r.log??(()=>{}),s=r.warn??a,d=r.targetId,K=r.parallel??3,U=r.timeoutMs??12e4,v=await f(`/api/eval-targets/${d}`),u=(await f(`/api/eval-targets/${d}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new w("no-labels",`No human labels on "${v.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:r.engineFlag??"auto",modelFlag:r.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new w("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(r.skipReviewed){let e=await f(`/api/eval-targets/${d}/judge-reviews`),t=new Set(e.filter(i=>W(i,n.engine,r.modelFlag)).map(i=>`${i.evalRunId}:${i.expectationIndex}`));if(b=u.filter(i=>!t.has(`${i.evalRunId}:${i.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await f(`/api/eval-targets/${d}/runs?limit=${C}&offset=${e}`);for(let i of t)y.set(i.id,i);if(t.length<C)break}let q=await f(`/api/eval-targets/${d}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let x=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let i=Z(t),g=P.get(e)??new Set;[...i].every(h=>g.has(h))||(o.delete(e),x+=1)}if(x>0&&a(`${x} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:x};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&B(e.error)){if(n.model===R)throw new w("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);s(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new w("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${v.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let $=[],I=0,k=new Set,m=r.limit!=null?[...o.entries()].slice(0,Math.max(1,r.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=p(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,i]=m[e],g=y.get(t),h=g?.grading?.expectations,O=g?.outputs?.transcript,F=g?X.get(g.evalId):void 0;if(!g||!h?.length||!O||!F){I+=i.length,s(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:F,output:O,expectations:h.map(l=>l.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let l of i){let L=h[l.expectationIndex],A=M.expectations[l.expectationIndex];if(!L||!A||L.text!==l.expectationText){I+=1;continue}if(M.judge_model==null&&A.method==="llm"){I+=1;continue}$.push({runId:t,expectationIndex:l.expectationIndex,text:l.expectationText,humanPassed:D(l.verdict,l.judgePassed),judgeAPassed:l.judgePassed,judgeBPassed:A.passed})}a(`  ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j($,I),_=[...k][0]??n.model??`${n.engine} (CLI default)`,E=0;if($.length>0)try{E=(await T(`/api/eval-targets/${d}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:$.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${E} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){s(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:v.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:E,alreadyReviewed:c,deferredPartialRuns:x}}p(oe,"runJudgeReviewPass");export{W as a,Z as b,w as c,oe as d};
+import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-KAWNGQS2.js";import{c as J}from"./chunk-K25S2MGU.js";import{b as B,c as R,d as N,e as z}from"./chunk-3BVHPQZW.js";import{e as f,g as T}from"./chunk-H2ORWIEY.js";import{a as p}from"./chunk-ZLLER3HP.js";function W(r,a,s){return s?r.reviewerModel===s||r.reviewerModel===`${a}:${s}`:r.reviewerEngine===a}p(W,"reviewMatchesReviewer");var C=100;function Z(r){let a=new Set;return r.forEach((s,d)=>{s.method!=="deterministic"&&a.add(d)}),a}p(Z,"llmExpectationIndexes");var w=class extends Error{constructor(s,d){super(d);this.reason=s}static{p(this,"JudgeReviewSkip")}};async function oe(r){let a=r.log??(()=>{}),s=r.warn??a,d=r.targetId,K=r.parallel??3,U=r.timeoutMs??12e4,v=await f(`/api/eval-targets/${d}`),u=(await f(`/api/eval-targets/${d}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new w("no-labels",`No human labels on "${v.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:r.engineFlag??"auto",modelFlag:r.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new w("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(r.skipReviewed){let e=await f(`/api/eval-targets/${d}/judge-reviews`),t=new Set(e.filter(i=>W(i,n.engine,r.modelFlag)).map(i=>`${i.evalRunId}:${i.expectationIndex}`));if(b=u.filter(i=>!t.has(`${i.evalRunId}:${i.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await f(`/api/eval-targets/${d}/runs?limit=${C}&offset=${e}`);for(let i of t)y.set(i.id,i);if(t.length<C)break}let q=await f(`/api/eval-targets/${d}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let x=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let i=Z(t),g=P.get(e)??new Set;[...i].every(h=>g.has(h))||(o.delete(e),x+=1)}if(x>0&&a(`${x} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:x};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&B(e.error)){if(n.model===R)throw new w("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);s(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new w("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${v.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let $=[],I=0,k=new Set,m=r.limit!=null?[...o.entries()].slice(0,Math.max(1,r.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=p(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,i]=m[e],g=y.get(t),h=g?.grading?.expectations,O=g?.outputs?.transcript,F=g?X.get(g.evalId):void 0;if(!g||!h?.length||!O||!F){I+=i.length,s(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:F,output:O,expectations:h.map(l=>l.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let l of i){let L=h[l.expectationIndex],A=M.expectations[l.expectationIndex];if(!L||!A||L.text!==l.expectationText){I+=1;continue}if(M.judge_model==null&&A.method==="llm"){I+=1;continue}$.push({runId:t,expectationIndex:l.expectationIndex,text:l.expectationText,humanPassed:D(l.verdict,l.judgePassed),judgeAPassed:l.judgePassed,judgeBPassed:A.passed})}a(`  ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j($,I),_=[...k][0]??n.model??`${n.engine} (CLI default)`,E=0;if($.length>0)try{E=(await T(`/api/eval-targets/${d}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:$.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${E} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){s(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:v.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:E,alreadyReviewed:c,deferredPartialRuns:x}}p(oe,"runJudgeReviewPass");export{W as a,Z as b,w as c,oe as d};

package/dist/chunk-HMMO27U4.js

@@ -0,0 +1 @@
+import{a as m}from"./chunk-ZLLER3HP.js";import{spawn as y}from"node:child_process";function O(r){try{let t=JSON.parse(r),i=t.result??t.content??r,a=t.usage?.total_tokens??(t.usage?.input_tokens??0)+(t.usage?.output_tokens??0),s;if(t.modelUsage&&typeof t.modelUsage=="object"){let o=-1;for(let[n,c]of Object.entries(t.modelUsage)){let u=typeof c?.outputTokens=="number"?c.outputTokens:0;u>o&&(o=u,s=n)}}return{output:i,totalTokens:a,model:s}}catch{return{output:r,totalTokens:0}}}m(O,"parseClaudeJsonOutput");function D(r){let{prompt:t,targetContent:i,systemPrompt:a,model:s}=r,o=["-p",t,"--output-format","json","--settings",JSON.stringify({disableAllHooks:!0})],n=i??a;return n&&o.push("--system-prompt",n),s&&o.push("--model",s),o}m(D,"buildClaudeArgs");async function S(r){let{prompt:t,targetContent:i,systemPrompt:a,workDir:s=process.cwd(),timeout:o=12e4,model:n}=r,c=D({prompt:t,targetContent:i,systemPrompt:a,model:n}),u=Date.now();return new Promise(d=>{let l={...process.env};delete l.CLAUDECODE,delete l.HTTPS_PROXY;let E=["/opt/homebrew/bin","/usr/local/bin"],f=(l.PATH??"").split(":").filter(Boolean),T=E.filter(e=>!f.includes(e));T.length&&(l.PATH=[...f,...T].join(":"));let p=y("claude",c,{cwd:s,env:l,stdio:["ignore","pipe","pipe"]}),g="",k="";p.stdout.on("data",e=>{g+=e.toString()}),p.stderr.on("data",e=>{k+=e.toString()});let x=setTimeout(()=>{p.kill("SIGTERM"),d({output:g,exitCode:null,durationMs:Date.now()-u,totalTokens:0,error:`Timed out after ${o}ms`})},o);p.on("close",e=>{clearTimeout(x);let b=Date.now()-u,{output:P,totalTokens:C,model:w}=O(g);d({output:P,exitCode:e,durationMs:b,totalTokens:C,model:w,error:e!==0?k||`Exit code ${e}`:void 0})}),p.on("error",e=>{clearTimeout(x),d({output:"",exitCode:null,durationMs:Date.now()-u,totalTokens:0,error:`Failed to spawn claude: ${e.message}`})})})}m(S,"executeEval");export{O as a,D as b,S as c};

package/dist/{chunk-YDRZDBHG.js → chunk-KAWNGQS2.js}

@@ -1,4 +1,4 @@
-import{a as w,b as A}from"./chunk-S6B2R2LL.js";import{c as b}from"./chunk-K25S2MGU.js";import{b as y}from"./chunk-WQ2BJQYB.js";import{a as c}from"./chunk-ZLLER3HP.js";var G=`You are a rigorous eval grader. Your job is to evaluate whether an AI assistant's output meets a set of expectations.
+import{a as S,b as _}from"./chunk-CDY7TQFZ.js";import{c as w}from"./chunk-K25S2MGU.js";import{c as A}from"./chunk-HMMO27U4.js";import{a as u}from"./chunk-ZLLER3HP.js";var E=`You are a rigorous eval grader. Your job is to evaluate whether an AI assistant's output meets a set of expectations.
 
 You will receive:
 1. The original prompt that was given to the AI
@@ -45,17 +45,17 @@ You MUST respond with ONLY a valid JSON object matching this exact structure (no
     "suggestions": [{ "assertion": "suggested new assertion", "reason": "why it matters" }],
     "overall": "brief summary assessment"
   }
-}`;async function I(S){let{prompt:_,output:m,expectations:g,model:u,timeout:f=12e4,engine:E="claude"}=S,r=new Array(g.length),o=[];g.forEach((e,i)=>{let t=w(e);if(t){let s=A(t,m);r[i]={text:e,passed:s.passed,evidence:s.evidence,method:"deterministic"}}else o.push({text:e,index:i})});let h=null,d=c((e,i)=>{let t=r.filter(l=>l.passed).length,s=r.length;return{expectations:r,summary:{passed:t,failed:s-t,total:s,pass_rate:s>0?t/s:0},claims:e,eval_feedback:i,judge_model:h}},"finish"),p=c((e,i)=>{for(let t of o)r[t.index]={text:t.text,passed:!1,evidence:e,method:"llm"};return d([],{suggestions:[],overall:i})},"failSubjective");if(o.length===0)return d([],{suggestions:[],overall:"All expectations verified deterministically \u2014 LLM judge skipped."});let v=`## Original Prompt
+}`;async function F(j){let{prompt:O,output:f,expectations:h,model:p,timeout:v=12e4,engine:x="claude"}=j,n=new Array(h.length),c=[];h.forEach((r,e)=>{let t=S(r);if(t){let i=_(t,f);n[e]={text:r,passed:i.passed,evidence:i.evidence,method:"deterministic",...i.errored?{errored:!0}:{}}}else c.push({text:r,index:e})});let m=null,g=u((r,e)=>{let t=n.filter(s=>!s.errored),i=n.length-t.length,o=t.filter(s=>s.passed).length,a=t.length;return{expectations:n,summary:{passed:o,failed:a-o,total:a,pass_rate:a>0?o/a:null,...i>0?{errored:i}:{}},claims:r,eval_feedback:e,judge_model:m}},"finish"),y=u((r,e)=>{for(let t of c)n[t.index]={text:t.text,passed:!1,evidence:r,method:"llm",errored:!0};return g([],{suggestions:[],overall:e})},"errorSubjective");if(c.length===0)return g([],{suggestions:[],overall:"All expectations verified deterministically \u2014 LLM judge skipped."});let b=`## Original Prompt
 
-${_}
+${O}
 
 ## AI Output
 
-${m}
+${f}
 
 ## Expectations to Evaluate
 
-${o.map((e,i)=>`${i+1}. ${e.text}`).join(`
+${c.map((r,e)=>`${e+1}. ${r.text}`).join(`
 `)}
 
-Grade the output against each expectation. Respond with ONLY valid JSON.`,a;if(E==="codex"){if(!u)return p("Grader error: the codex engine requires an explicit --model","Grading failed: codex engine requires an explicit model");a=await b({prompt:v,systemPrompt:G,model:u,timeout:f})}else a=await y({prompt:v,targetContent:G,timeout:f,model:u});if(h=a.model??null,a.error)return p(`Grader error: ${a.error}`,`Grading failed: ${a.error}`);try{let e=a.output.trim(),i=e.match(/```(?:json)?\s*([\s\S]*?)```/);i&&(e=i[1].trim());let t=e.match(/\{[\s\S]*\}/);t&&(e=t[0]);let s=JSON.parse(e);return o.forEach((l,j)=>{let n=s.expectations?.[j],x=n?.flag==="non_discriminating"||n?.flag==="unverifiable"?n.flag:void 0;r[l.index]=n?{text:l.text,passed:n.passed,evidence:n.evidence??"",method:"llm",...x?{flag:x}:{}}:{text:l.text,passed:!1,evidence:"Judge did not return a grade for this expectation",method:"llm"}}),d(s.claims??[],s.eval_feedback??{suggestions:[],overall:""})}catch{return p("Grader output was not valid JSON",`Grader returned unparseable output: ${a.output.slice(0,200)}`)}}c(I,"gradeEvalOutput");export{I as a};
+Grade the output against each expectation. Respond with ONLY valid JSON.`;if(x==="codex"&&!p)return y("Grader error: the codex engine requires an explicit --model","Grading failed: codex engine requires an explicit model");let k=2,d={evidence:"Grader error: judge did not run",overall:"Grading failed"};for(let r=1;r<=k;r++){let e;if(x==="codex"?e=await w({prompt:b,systemPrompt:E,model:p,timeout:v}):e=await A({prompt:b,targetContent:E,timeout:v,model:p}),m=e.model??m,e.error){d={evidence:`Grader error: ${e.error}`,overall:`Grading failed: ${e.error}`};continue}try{let t=e.output.trim(),i=t.match(/```(?:json)?\s*([\s\S]*?)```/);i&&(t=i[1].trim());let o=t.match(/\{[\s\S]*\}/);o&&(t=o[0]);let a=JSON.parse(t);return c.forEach((s,R)=>{let l=a.expectations?.[R],G=l?.flag==="non_discriminating"||l?.flag==="unverifiable"?l.flag:void 0;n[s.index]=l?{text:s.text,passed:l.passed,evidence:l.evidence??"",method:"llm",...G?{flag:G}:{}}:{text:s.text,passed:!1,evidence:"Judge did not return a grade for this expectation",method:"llm",errored:!0}}),g(a.claims??[],a.eval_feedback??{suggestions:[],overall:""})}catch{d={evidence:"Grader output was not valid JSON",overall:`Grader returned unparseable output: ${e.output.slice(0,200)}`}}}return y(d.evidence,d.overall)}u(F,"gradeEvalOutput");export{F as a};

package/dist/chunk-L2CFSMBC.js

@@ -0,0 +1 @@
+import{a as d}from"./chunk-Z72VHGV7.js";import{a}from"./chunk-ZLLER3HP.js";import{mkdirSync as y,readdirSync as v,rmSync as D,writeFileSync as E}from"node:fs";import{join as c,relative as w,sep as S}from"node:path";var h="v2-neutral-workspace",A=["CLAUDE.md","CLAUDE.local.md","AGENTS.md"],C=new Set(["node_modules",".git","dist","build",".next","coverage"]);function m(e){let r=[],s=new Set(A),t=a(l=>{let n;try{n=v(l,{withFileTypes:!0})}catch{return}for(let i of n){let o=c(l,i.name);if(i.isDirectory()){if(C.has(i.name)||i.name.startsWith(".")&&i.name!==".claude")continue;t(o)}else s.has(i.name)&&r.push(w(e,o))}},"walk");return t(e),r}a(m,"findAmbientInstructionFiles");function I(e){let r=[];for(let s of m(e))try{D(c(e,s),{force:!0}),r.push(s)}catch{}return r}a(I,"stripAmbientInstructions");function W(e){return(e??"").toLowerCase().replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"")||"eval-target-skill"}a(W,"skillSlug");function b(e={}){let{baseDir:r,targetType:s,targetContent:t,targetName:l}=e,n=d(r),i=I(n.workDir),o="none",p,u=[];if(t)switch(s){case"prompt":p=t,o="system_prompt";break;case"skill":{g(n.workDir,c(".claude","skills",W(l),"SKILL.md"),t),o="skill";break}default:g(n.workDir,"CLAUDE.md",t),u.push("CLAUDE.md"),o="claude_md";break}let k=m(n.workDir).filter(f=>!u.includes(f));return{workDir:n.workDir,systemPrompt:p,delivery:o,methodology:h,isWorktree:n.isWorktree,strippedFiles:i,leakedFiles:k,cleanup:n.cleanup}}a(b,"createEvalWorkspace");function g(e,r,s){let t=c(e,r),l=t.slice(0,t.lastIndexOf(S));y(l,{recursive:!0}),E(t,s,"utf8")}a(g,"writeFileInWorkspace");export{h as a,A as b,m as c,W as d,b as e};

package/dist/{chunk-ARZJNTMV.js → chunk-M4YF7YO7.js}

@@ -1,2 +1,2 @@
-import{c as R}from"./chunk-6QKI756T.js";import{a as P}from"./chunk-5RBLIVBF.js";import{b as s,c as r,j}from"./chunk-KZ65LEUH.js";import{a as g}from"./chunk-ZLLER3HP.js";import{and as y,eq as l,isNotNull as I}from"drizzle-orm";async function O($,p={}){let k=p.db??j(),b=p.removeWorktreeFn??R,n=p.log??(e=>process.stderr.write(`[process_run_cleanup] ${e}
+import{c as R}from"./chunk-MDPN46EY.js";import{a as P}from"./chunk-5RBLIVBF.js";import{b as s,c as r,j}from"./chunk-KZ65LEUH.js";import{a as g}from"./chunk-ZLLER3HP.js";import{and as y,eq as l,isNotNull as I}from"drizzle-orm";async function O($,p={}){let k=p.db??j(),b=p.removeWorktreeFn??R,n=p.log??(e=>process.stderr.write(`[process_run_cleanup] ${e}
 `)),{runId:t}=$,[i]=await k.select({id:s.id,userId:s.userId,projectPath:s.projectPath}).from(s).where(l(s.id,t)).limit(1);if(!i)throw new P(`Run ${t} not found`);if(!i.projectPath)throw new P(`Run ${t} has no projectPath; cannot resolve a repo root for worktree cleanup`);let c=i.projectPath,v=await k.select({id:r.id,projectPath:r.projectPath}).from(r).where(y(l(r.runId,t),l(r.userId,i.userId),I(r.endedAt))),a=[],u=0,d=0,h=0,m=0,w=new Set;for(let e of v)if(!(!e.projectPath||e.projectPath===c)&&!w.has(e.projectPath)){w.add(e.projectPath);try{let o=await b({repoRoot:c,worktreePath:e.projectPath});o.removed?(u++,a.push({sessionId:e.id,worktreePath:e.projectPath,status:"removed"}),n(`removed worktree ${e.projectPath} (session ${e.id})`)):o.reason==="dirty"?(d++,a.push({sessionId:e.id,worktreePath:e.projectPath,status:"skipped-dirty"}),n(`skipped dirty worktree ${e.projectPath} (session ${e.id}) \u2014 uncommitted changes`)):(h++,a.push({sessionId:e.id,worktreePath:e.projectPath,status:"absent"}),n(`worktree absent ${e.projectPath} (session ${e.id})`))}catch(o){let f=o instanceof Error?o.message:String(o);m++,a.push({sessionId:e.id,worktreePath:e.projectPath,status:"error",error:f.slice(0,1e3)}),n(`error removing worktree ${e.projectPath} (session ${e.id}): ${f}`)}}return n(`run ${t} cleanup: removed=${u} skippedDirty=${d} absent=${h} errored=${m}`),{runId:t,repoRoot:c,removed:u,skippedDirty:d,absent:h,errored:m,worktrees:a}}g(O,"processRunCleanup");export{O as a};

package/dist/{chunk-6QKI756T.js → chunk-MDPN46EY.js}

@@ -1,2 +1,2 @@
-import{y as w}from"./chunk-7DFNBRLY.js";import{a as m}from"./chunk-N7ODX6PO.js";import{a as s}from"./chunk-ZLLER3HP.js";import{spawn as g}from"node:child_process";import d from"node:fs";import b from"node:path";function c(e,t){return new Promise((r,n)=>{let o=g("git",e,{cwd:t,stdio:["ignore","pipe","pipe"],env:m()}),i="",f="";o.stdout.on("data",a=>{i+=a.toString()}),o.stderr.on("data",a=>{f+=a.toString()}),o.on("error",a=>n(new Error(`git ${e.join(" ")} failed to spawn: ${a.message}`))),o.on("close",a=>{a===0?r({code:a,stdout:i,stderr:f}):n(new Error(`git ${e.join(" ")} exited ${a}: ${f.trim()||i.trim()}`))})})}s(c,"runGit");async function k(e,t){if(!t||!t.trim())throw new Error('invalid branch name "": branch must be a non-empty git refname');if(t.startsWith("-"))throw new Error(`invalid branch name "${t}": must not start with "-" (git would parse it as an option)`);if(/\s/.test(t))throw new Error(`invalid branch name "${t}": must not contain whitespace`);try{await c(["check-ref-format","--branch",t],e)}catch{throw new Error(`invalid branch name "${t}": not a valid git refname`)}}s(k,"assertValidBranch");function h(e){let t=b.resolve(e);try{return d.realpathSync(t)}catch{return t}}s(h,"canonical");function y(e){let t=[],r=null;for(let n of e.split(`
+import{y as w}from"./chunk-5DIOZDDZ.js";import{a as m}from"./chunk-N7ODX6PO.js";import{a as s}from"./chunk-ZLLER3HP.js";import{spawn as g}from"node:child_process";import d from"node:fs";import b from"node:path";function c(e,t){return new Promise((r,n)=>{let o=g("git",e,{cwd:t,stdio:["ignore","pipe","pipe"],env:m()}),i="",f="";o.stdout.on("data",a=>{i+=a.toString()}),o.stderr.on("data",a=>{f+=a.toString()}),o.on("error",a=>n(new Error(`git ${e.join(" ")} failed to spawn: ${a.message}`))),o.on("close",a=>{a===0?r({code:a,stdout:i,stderr:f}):n(new Error(`git ${e.join(" ")} exited ${a}: ${f.trim()||i.trim()}`))})})}s(c,"runGit");async function k(e,t){if(!t||!t.trim())throw new Error('invalid branch name "": branch must be a non-empty git refname');if(t.startsWith("-"))throw new Error(`invalid branch name "${t}": must not start with "-" (git would parse it as an option)`);if(/\s/.test(t))throw new Error(`invalid branch name "${t}": must not contain whitespace`);try{await c(["check-ref-format","--branch",t],e)}catch{throw new Error(`invalid branch name "${t}": not a valid git refname`)}}s(k,"assertValidBranch");function h(e){let t=b.resolve(e);try{return d.realpathSync(t)}catch{return t}}s(h,"canonical");function y(e){let t=[],r=null;for(let n of e.split(`
 `))n.startsWith("worktree ")?(r&&t.push(r),r={path:n.slice(9).trim()}):n.startsWith("branch ")&&r&&(r.branch=n.slice(7).trim().replace(/^refs\/heads\//,""));return r&&t.push(r),t}s(y,"parseWorktreeList");async function v(e,t){try{return await c(["show-ref","--verify","--quiet",`refs/heads/${t}`],e),!0}catch{return!1}}s(v,"branchExists");async function B(e){let{repoRoot:t,branch:r,baseRef:n,allowExistingBranch:o=!1}=e;await k(t,r);let i=w({repoRoot:t,branch:r,basePath:e.basePath}),f=await c(["worktree","list","--porcelain"],t),a=y(f.stdout),p=a.find(u=>u.branch===r);if(p)return{path:h(p.path),branch:r,created:!1};let l=a.find(u=>h(u.path)===i);if(l)return{path:h(l.path),branch:l.branch??r,created:!1};if(await v(t,r)){if(!o)throw new Error(`branch "${r}" already exists with no worktree; refusing to clobber (pass allowExistingBranch:true to attach a worktree onto it)`);return await c(["worktree","add",i,r],t),{path:h(i),branch:r,created:!0}}return await c(["worktree","add",i,"-b",r,n],t),{path:h(i),branch:r,created:!0}}s(B,"ensureWorktree");async function S(e){let{repoRoot:t,worktreePath:r}=e;if(!d.existsSync(r))return{removed:!1,reason:"absent"};let n=!1;try{n=(await c(["status","--porcelain"],r)).stdout.trim().length>0}catch{n=!1}if(n)return{removed:!1,reason:"dirty"};try{return await c(["worktree","remove",r],t),{removed:!0}}catch(o){let i=o instanceof Error?o.message:String(o);if(/not a working tree|No such file or directory|not a valid path|is not a working tree/i.test(i))return{removed:!1,reason:"absent"};throw o}}s(S,"removeWorktree");export{c as a,B as b,S as c};

package/dist/{chunk-MYPDODRJ.js → chunk-OSIP6FUD.js}

@@ -1 +1 @@
-import{a as i}from"./chunk-JUF4JLZQ.js";import{h as s}from"./chunk-H2ORWIEY.js";import{a as n}from"./chunk-ZLLER3HP.js";async function l(a,c){let{mirrorId:g,evalTargetId:p,config:r}=a,e=n(o=>c.logger.info(o),"log");await i(r,p,e);let t={type:"conversation",conversationId:r.conversationId};return await s(`/api/eval-jobs/${g}/state`,{result:t}).catch(o=>{e(`result persist failed: ${o instanceof Error?o.message:String(o)}`)}),t}n(l,"processConversation");export{l as a};
+import{a as i}from"./chunk-QQ2UPNBC.js";import{h as s}from"./chunk-H2ORWIEY.js";import{a as n}from"./chunk-ZLLER3HP.js";async function l(a,c){let{mirrorId:g,evalTargetId:p,config:r}=a,e=n(o=>c.logger.info(o),"log");await i(r,p,e);let t={type:"conversation",conversationId:r.conversationId};return await s(`/api/eval-jobs/${g}/state`,{result:t}).catch(o=>{e(`result persist failed: ${o instanceof Error?o.message:String(o)}`)}),t}n(l,"processConversation");export{l as a};

package/dist/{chunk-JUF4JLZQ.js → chunk-QQ2UPNBC.js}

@@ -1,4 +1,4 @@
-import{a as C}from"./chunk-GQRCSR2A.js";import{b as $}from"./chunk-UUMW6X2W.js";import{a as y}from"./chunk-YDRZDBHG.js";import{b as f}from"./chunk-WQ2BJQYB.js";import{e as h,f as w}from"./chunk-H2ORWIEY.js";import{a as v}from"./chunk-ZLLER3HP.js";async function N(n,i,p){let{conversationId:a,messageId:g,messageContent:r}=n,o=await h(`/api/eval-targets/${i}`),c=await h(`/api/eval-targets/${i}/evals`),e=await h(`/api/eval-conversations/${a}/messages`),t=e.findIndex(l=>l.id===g);if(t>=0&&e.slice(t+1).some(u=>u.role==="assistant"&&(u.type==="results"||u.type==="proposal"||u.type==="generated"||u.type==="text"))){p(`Message ${g} already processed \u2014 skipping (idempotency guard)`);return}let s=v(async(l,u,m)=>{await w(`/api/eval-conversations/${a}/messages/assistant`,{content:l,type:u,data:m})},"postMessage"),d=await _(o,c,e,r,p);p(`Action: ${d.action}`);try{switch(d.action){case"run_evals":await x(o,c,a,s,p);break;case"generate_cases":await k(o,i,a,s,p);break;case"analyze_results":case"propose_changes":await E(o,c,e,r,a,s,p);break;case"respond":await s(d.response,"text");break}}catch(l){let u=l instanceof Error?l.message:String(l);throw await s(`Something went wrong: ${u}`,"text").catch(()=>{}),l}}v(N,"processConversationJob");async function _(n,i,p,a,g){let r=p.slice(-10).map(e=>`${e.role}: ${e.content.slice(0,200)}`).join(`
+import{a as C}from"./chunk-4VZLIGMS.js";import{b as $}from"./chunk-UUMW6X2W.js";import{a as y}from"./chunk-KAWNGQS2.js";import{c as f}from"./chunk-HMMO27U4.js";import{e as h,f as w}from"./chunk-H2ORWIEY.js";import{a as v}from"./chunk-ZLLER3HP.js";async function N(n,i,p){let{conversationId:a,messageId:g,messageContent:r}=n,o=await h(`/api/eval-targets/${i}`),c=await h(`/api/eval-targets/${i}/evals`),e=await h(`/api/eval-conversations/${a}/messages`),t=e.findIndex(l=>l.id===g);if(t>=0&&e.slice(t+1).some(u=>u.role==="assistant"&&(u.type==="results"||u.type==="proposal"||u.type==="generated"||u.type==="text"))){p(`Message ${g} already processed \u2014 skipping (idempotency guard)`);return}let s=v(async(l,u,m)=>{await w(`/api/eval-conversations/${a}/messages/assistant`,{content:l,type:u,data:m})},"postMessage"),d=await _(o,c,e,r,p);p(`Action: ${d.action}`);try{switch(d.action){case"run_evals":await x(o,c,a,s,p);break;case"generate_cases":await k(o,i,a,s,p);break;case"analyze_results":case"propose_changes":await E(o,c,e,r,a,s,p);break;case"respond":await s(d.response,"text");break}}catch(l){let u=l instanceof Error?l.message:String(l);throw await s(`Something went wrong: ${u}`,"text").catch(()=>{}),l}}v(N,"processConversationJob");async function _(n,i,p,a,g){let r=p.slice(-10).map(e=>`${e.role}: ${e.content.slice(0,200)}`).join(`
 `),o=`You are an eval assistant. Decide what action to take based on the user's message.
 
 ## Current State

package/dist/{chunk-KG3TW5QB.js → chunk-T4P2NV3P.js}

@@ -1 +1 @@
-import{c as d,d as t}from"./chunk-BSXUH5FD.js";import{a as r}from"./chunk-ZLLER3HP.js";async function g(o,i){let{evalTargetId:s}=o;try{let e=await t({targetId:s,engineFlag:"auto",skipReviewed:!0,log:r(a=>i.logger.info(`[judge-review] ${a}`),"log"),warn:r(a=>i.logger.warn(`[judge-review] ${a}`),"warn")});return{evalTargetId:s,reviewerModel:e.reviewerModel,comparedPairs:e.report.pairs,disagreements:e.report.disagreements.length,persisted:e.persisted,alreadyReviewed:e.alreadyReviewed,deferredPartialRuns:e.deferredPartialRuns}}catch(e){if(e instanceof d)return i.logger.info(`[judge-review] skipped (${e.reason}): ${e.message}`),{evalTargetId:s,skipped:e.reason};throw e}}r(g,"processJudgeReview");export{g as a};
+import{c as d,d as t}from"./chunk-HL755A5U.js";import{a as r}from"./chunk-ZLLER3HP.js";async function g(o,i){let{evalTargetId:s}=o;try{let e=await t({targetId:s,engineFlag:"auto",skipReviewed:!0,log:r(a=>i.logger.info(`[judge-review] ${a}`),"log"),warn:r(a=>i.logger.warn(`[judge-review] ${a}`),"warn")});return{evalTargetId:s,reviewerModel:e.reviewerModel,comparedPairs:e.report.pairs,disagreements:e.report.disagreements.length,persisted:e.persisted,alreadyReviewed:e.alreadyReviewed,deferredPartialRuns:e.deferredPartialRuns}}catch(e){if(e instanceof d)return i.logger.info(`[judge-review] skipped (${e.reason}): ${e.message}`),{evalTargetId:s,skipped:e.reason};throw e}}r(g,"processJudgeReview");export{g as a};

package/dist/chunk-ULRVMH7O.js

@@ -0,0 +1 @@
+import{a as M}from"./chunk-5RBLIVBF.js";import{a as T}from"./chunk-YQ56FQEO.js";import{b as A}from"./chunk-UUMW6X2W.js";import{e as S}from"./chunk-L2CFSMBC.js";import{a as x}from"./chunk-KAWNGQS2.js";import{c as C}from"./chunk-HMMO27U4.js";import{e as k,f as E,h as p}from"./chunk-H2ORWIEY.js";import{a as w}from"./chunk-ZLLER3HP.js";async function W(v,f){let{mirrorId:i,evalTargetId:g,config:o}=v,a=w(e=>f.logger.info(e),"log"),s=await k(`/api/eval-targets/${g}`),$=await k(`/api/eval-targets/${g}/evals`);if($.length===0&&o.generateIfEmpty){a("No eval cases on target \u2014 generating from content via local LLM...");let e=await A({content:s.content,name:s.name});if(e===null)throw new Error("Case generation requires an LLM backend (claude CLI on PATH or ANTHROPIC_API_KEY) \u2014 none available on this worker.");let n=[];for(let t of e)n.push(await E(`/api/eval-targets/${g}/evals`,t));$=n,a(`Generated ${n.length} eval case(s)`)}if($.length===0)throw new M("No evals defined for target");let G=o.runsPerEval??1,R=o.timeout??12e4,j=o.iteration??1,y=new Set;if(f.job.attempts>1)for(let n=0;;n+=100){let t=await k(`/api/eval-targets/${g}/runs?jobId=${i}&limit=100&offset=${n}`);for(let r of t)y.add(`${r.evalId}|${r.configuration}|${r.runNumber}|${r.iteration}`);if(t.length<100)break}let _=[{name:"with_target",content:s.content}];o.skipBaseline||_.push({name:"without_target"});let d=[];for(let e of $)for(let n of _)for(let t=1;t<=G;t++)y.has(`${e.id}|${n.name}|${t}|${j}`)||d.push({eval_:e,configuration:n.name,targetContent:n.content,runNumber:t});y.size>0&&a(`Resuming attempt ${f.job.attempts}: ${y.size} run(s) already persisted, ${d.length} remaining`);let l=d.length,u=0,m=Math.min(o.parallel??1,4);await p(`/api/eval-jobs/${i}/progress`,{status:"running",progress:{total:l,completed:0,current:null,message:`Starting ${l} eval runs (concurrency: ${m})...`}});let b=[];async function I(e){let n=`${e.configuration} \xB7 ${e.eval_.name} \xB7 run#${e.runNumber}`;a(`[${u+1}/${l}] ${n}`);let t=S({targetType:s.type,targetContent:e.targetContent,targetName:s.name});t.leakedFiles.length&&a(`\u26A0 CONTAMINATED ${n}: ${t.leakedFiles.join(", ")} survived the strip`);try{let r=await C({prompt:e.eval_.prompt,targetContent:t.systemPrompt,workDir:t.workDir,timeout:R,model:o.model}),c={};return!o.skipGrading&&e.eval_.expectations.length>0&&!r.error&&(c=await x({prompt:e.eval_.prompt,output:r.output,expectations:e.eval_.expectations,model:o.graderModel??o.model,timeout:R})),{jobId:i,evalId:e.eval_.id,evalTargetId:s.id,evalTargetVersion:s.version,configuration:e.configuration,runNumber:e.runNumber,iteration:j,status:r.error?"failed":"completed",outputs:{...r.error?{error:r.error}:{transcript:r.output,...r.model?{model:r.model}:{}},methodology:t.methodology,delivery:t.delivery,isolated:t.isWorktree,...t.leakedFiles.length?{leakedFiles:t.leakedFiles}:{}},timing:{duration_ms:r.durationMs,total_tokens:r.totalTokens},grading:c,worktree:t.isWorktree?!0:void 0}}finally{t.cleanup()}}if(w(I,"processItem"),m<=1)for(let e of d){let n=`${e.configuration} \xB7 ${e.eval_.name} \xB7 run#${e.runNumber}`;await p(`/api/eval-jobs/${i}/progress`,{status:"running",progress:{total:l,completed:u,current:n,message:`Executing ${n}...`}});let t=await I(e);await E("/api/eval-runs",t),b.push(t),u++,await p(`/api/eval-jobs/${i}/progress`,{status:"running",progress:{total:l,completed:u,current:n,message:`Completed ${n}`}})}else{a(`Running ${l} evals with concurrency=${m} (worktree isolation)`);for(let e=0;e<d.length;e+=m){let n=d.slice(e,e+m),t=`batch ${Math.floor(e/m)+1}/${Math.ceil(d.length/m)}`;await p(`/api/eval-jobs/${i}/progress`,{status:"running",progress:{total:l,completed:u,current:t,message:`Executing ${t} (${n.length} parallel)...`}});let r=await Promise.all(n.map(I));for(let c of r)await E("/api/eval-runs",c),b.push(c),u++;await p(`/api/eval-jobs/${i}/progress`,{status:"running",progress:{total:l,completed:u,current:t,message:`Completed ${t}`}})}}let P=J(b),h;if(!o.skipGrading&&b.some(e=>e.grading&&Object.keys(e.grading).length>0))try{a(`Running meta-judge for job ${i}...`);let e=$.map(n=>{let r=b.filter(c=>c.evalId===n.id&&c.configuration==="with_target")[0];return{evalName:n.name,prompt:n.prompt,expectations:n.expectations,grading:r?.grading?r.grading:null,output:r?.outputs?.transcript?.slice(0,500)}});h=await T({targetName:s.name,evalResults:e,model:o.graderModel??o.model,timeout:6e4}),a(`Meta-judge: quality=${h.quality_score}/5, issues=${h.issues.length}`)}catch(e){a(`Meta-judge failed (non-fatal): ${e instanceof Error?e.message:String(e)}`)}let N=h?{...P,meta_judgement:h}:P;return a(`Job ${i} completed: ${u}/${l} runs`),await p(`/api/eval-jobs/${i}/state`,{result:N}).catch(e=>{a(`result persist failed: ${e instanceof Error?e.message:String(e)}`)}),N}w(W,"processEval");function J(v){let f=v.filter(a=>a.status==="completed"),i=v.filter(a=>a.status==="failed"),g=f.filter(a=>{let s=a.grading;return s&&Object.keys(s).length>0}),o=g.length>0?g.reduce((a,s)=>a+(s.grading?.summary?.pass_rate??0),0)/g.length:null;return{total:v.length,completed:f.length,failed:i.length,avgPassRate:o}}w(J,"buildSummary");export{W as a};

package/dist/{chunk-DMLAMJLV.js → chunk-WYF5BIQW.js}

@@ -1 +1 @@
-import{a as l}from"./chunk-CHA5SX4Y.js";import{a as r}from"./chunk-ZLLER3HP.js";var u=r(n=>{let t=n.grading.summary?.pass_rate;return typeof t=="number"?t:null},"passRateOf"),a=r(n=>(u(n)??0)>=1,"fullPass");function f(n,t){let i=new Map(n.filter(e=>u(e)!=null).map(e=>[e.evalName,e])),o=[];for(let e of t){if(u(e)==null)continue;let s=i.get(e.evalName);s&&o.push({withScore:a(e)?1:0,withoutScore:a(s)?1:0})}return o}r(f,"pairVersionResults");function d(n,t){let i=f(n,t);return i.length===0?null:l(i,{baseline:"prior-version"})}r(d,"buildVersionVerdict");function m(n,t,i){let o=`v${i} vs v${t}`;if(!n.sufficient)return`${o}: ${n.insufficientReason??"insufficient data"} \u2014 no effect claimed`;let e=n.delta>=0?"+":"",s=`95% CI ${(n.ci.lower*100).toFixed(0)}\u2013${(n.ci.upper*100).toFixed(0)}%`,c=n.significant?"significant":"not significant";return`${o}: ${e}${(n.delta*100).toFixed(0)}% pass rate (${s}, n=${n.nWith} paired, ${c})`}r(m,"formatVersionVerdict");export{f as a,d as b,m as c};
+import{a as l}from"./chunk-X2ILN3KP.js";import{a as r}from"./chunk-ZLLER3HP.js";var u=r(n=>{let t=n.grading.summary?.pass_rate;return typeof t=="number"?t:null},"passRateOf"),a=r(n=>(u(n)??0)>=1,"fullPass");function f(n,t){let i=new Map(n.filter(e=>u(e)!=null).map(e=>[e.evalName,e])),o=[];for(let e of t){if(u(e)==null)continue;let s=i.get(e.evalName);s&&o.push({withScore:a(e)?1:0,withoutScore:a(s)?1:0})}return o}r(f,"pairVersionResults");function d(n,t){let i=f(n,t);return i.length===0?null:l(i,{baseline:"prior-version"})}r(d,"buildVersionVerdict");function m(n,t,i){let o=`v${i} vs v${t}`;if(!n.sufficient)return`${o}: ${n.insufficientReason??"insufficient data"} \u2014 no effect claimed`;let e=n.delta>=0?"+":"",s=`95% CI ${(n.ci.lower*100).toFixed(0)}\u2013${(n.ci.upper*100).toFixed(0)}%`,c=n.significant?"significant":"not significant";return`${o}: ${e}${(n.delta*100).toFixed(0)}% pass rate (${s}, n=${n.nWith} paired, ${c})`}r(m,"formatVersionVerdict");export{f as a,d as b,m as c};

package/dist/{chunk-CHA5SX4Y.js → chunk-X2ILN3KP.js}

@@ -1 +1 @@
-import{a as r}from"./chunk-ZLLER3HP.js";function g(e,o){let t=e,i=o,n=t+i;if(n===0)return{pValue:1,nDiscordant:0};let c=Math.min(t,i),d=Math.pow(.5,n),s=d;for(let a=1;a<=c;a++)d*=(n-a+1)/a,s+=d;return{pValue:Math.min(1,2*s),nDiscordant:n}}r(g,"mcnemarExact");function w(e,o,t=10){return{sufficient:e>=t&&o>=t,minN:t,nWith:e,nWithout:o,shortfall:{with:Math.max(0,t-e),without:Math.max(0,t-o)}}}r(w,"assessSufficiency");function R(e){let o=e>>>0;return()=>{o=o+1831565813|0;let t=Math.imul(o^o>>>15,1|o);return t=t+Math.imul(t^t>>>7,61|t)^t,((t^t>>>14)>>>0)/4294967296}}r(R,"mulberry32");function b(e,o){let t=e.length;if(t===1)return e[0];let i=(t-1)*o,n=Math.floor(i),c=Math.ceil(i);return n===c?e[n]:e[n]+(i-n)*(e[c]-e[n])}r(b,"sortedQuantile");function x(e,o={}){let t=e.length;if(t===0)return{estimate:0,lower:0,upper:0,iterations:0};let i=o.iterations??1e4,n=o.ciLevel??.95,c=R(o.seed??1),d=e.reduce((u,f)=>u+f,0)/t,s=new Array(i);for(let u=0;u<i;u++){let f=0;for(let h=0;h<t;h++)f+=e[c()*t|0];s[u]=f/t}s.sort((u,f)=>u-f);let a=(1-n)/2;return{estimate:d,lower:b(s,a),upper:b(s,1-a),iterations:i}}r(x,"pairedBootstrapCI");var M=r(e=>e===0||e===1,"isBinary");function A(e,o,t){return e.every(n=>n[o]!=null&&n[t]!=null)?e.reduce((n,c)=>n+(c[o]-c[t]),0)/e.length:null}r(A,"meanPairDelta");function N(e,o={}){let t=o.baseline??"no-rule",i=o.minN??10,n=e.length,c=e.map(l=>l.withScore-l.withoutScore),d=n===0?0:c.reduce((l,m)=>l+m,0)/n,s=x(c,{seed:o.seed,iterations:o.iterations}),a={lower:s.lower,upper:s.upper},u=e.every(l=>M(l.withScore)&&M(l.withoutScore)),f=null;if(u&&n>0){let l=0,m=0;for(let p of e)p.withScore===1&&p.withoutScore===0?l++:p.withScore===0&&p.withoutScore===1&&m++;f=g(l,m).pValue}let{sufficient:h}=w(n,n,i),S=a.lower>0||a.upper<0,v={mode:"controlled",baseline:t,delta:d,ci:a,pValue:f,significant:h&&S,sufficient:h,nWith:n,nWithout:n,costDelta:A(e,"withCost","withoutCost"),latencyDelta:A(e,"withLatency","withoutLatency"),confounded:!1};return h||(v.insufficientReason=`insufficient data: need >= ${i} paired tasks per cohort, have ${n}`),v}r(N,"buildControlledVerdict");export{N as a};
+import{a as r}from"./chunk-ZLLER3HP.js";function g(e,o){let t=e,i=o,n=t+i;if(n===0)return{pValue:1,nDiscordant:0};let c=Math.min(t,i),d=Math.pow(.5,n),s=d;for(let a=1;a<=c;a++)d*=(n-a+1)/a,s+=d;return{pValue:Math.min(1,2*s),nDiscordant:n}}r(g,"mcnemarExact");function w(e,o,t=10){return{sufficient:e>=t&&o>=t,minN:t,nWith:e,nWithout:o,shortfall:{with:Math.max(0,t-e),without:Math.max(0,t-o)}}}r(w,"assessSufficiency");function R(e){let o=e>>>0;return()=>{o=o+1831565813|0;let t=Math.imul(o^o>>>15,1|o);return t=t+Math.imul(t^t>>>7,61|t)^t,((t^t>>>14)>>>0)/4294967296}}r(R,"mulberry32");function b(e,o){let t=e.length;if(t===1)return e[0];let i=(t-1)*o,n=Math.floor(i),c=Math.ceil(i);return n===c?e[n]:e[n]+(i-n)*(e[c]-e[n])}r(b,"sortedQuantile");function x(e,o={}){let t=e.length;if(t===0)return{estimate:0,lower:0,upper:0,iterations:0};let i=o.iterations??1e4,n=o.ciLevel??.95,c=R(o.seed??1),d=e.reduce((u,f)=>u+f,0)/t,s=new Array(i);for(let u=0;u<i;u++){let f=0;for(let h=0;h<t;h++)f+=e[c()*t|0];s[u]=f/t}s.sort((u,f)=>u-f);let a=(1-n)/2;return{estimate:d,lower:b(s,a),upper:b(s,1-a),iterations:i}}r(x,"pairedBootstrapCI");var M=r(e=>e===0||e===1,"isBinary");function A(e,o,t){return e.every(n=>n[o]!=null&&n[t]!=null)?e.reduce((n,c)=>n+(c[o]-c[t]),0)/e.length:null}r(A,"meanPairDelta");function T(e,o={}){let t=o.baseline??"no-rule",i=o.minN??10,n=e.length,c=e.map(l=>l.withScore-l.withoutScore),d=n===0?0:c.reduce((l,m)=>l+m,0)/n,s=x(c,{seed:o.seed,iterations:o.iterations}),a={lower:s.lower,upper:s.upper},u=e.every(l=>M(l.withScore)&&M(l.withoutScore)),f=null;if(u&&n>0){let l=0,m=0;for(let p of e)p.withScore===1&&p.withoutScore===0?l++:p.withScore===0&&p.withoutScore===1&&m++;f=g(l,m).pValue}let{sufficient:h}=w(n,n,i),_=a.lower>0||a.upper<0,v={mode:"controlled",baseline:t,delta:d,ci:a,pValue:f,significant:h&&_,sufficient:h,nWith:n,nWithout:n,costDelta:A(e,"withCost","withoutCost"),latencyDelta:A(e,"withLatency","withoutLatency"),confounded:!1};return h||(v.insufficientReason=`insufficient data: need >= ${i} paired tasks per cohort, have ${n}`),v}r(T,"buildControlledVerdict");export{T as a};

package/dist/{chunk-G6XGDHGC.js → chunk-XA4KSQEC.js}

@@ -1,2 +1,2 @@
-import{c as E,o as m}from"./chunk-7DFNBRLY.js";import{a as d}from"./chunk-H2ORWIEY.js";import{a as c}from"./chunk-ZLLER3HP.js";import*as p from"node:fs";import*as u from"node:path";import*as _ from"node:os";function T(){if(process.env.RULEMETRIC_API_KEY||process.env.RULEMETRIC_ACCESS_TOKEN)return;let s=u.join(_.homedir(),".config","rulemetric","env");C(s)||C(u.join(process.cwd(),".env.local"))}c(T,"ensureEnv");function C(s){try{let t=p.readFileSync(s,"utf-8");for(let n of t.split(`
+import{c as E,o as m}from"./chunk-5DIOZDDZ.js";import{a as d}from"./chunk-H2ORWIEY.js";import{a as c}from"./chunk-ZLLER3HP.js";import*as p from"node:fs";import*as u from"node:path";import*as _ from"node:os";function T(){if(process.env.RULEMETRIC_API_KEY||process.env.RULEMETRIC_ACCESS_TOKEN)return;let s=u.join(_.homedir(),".config","rulemetric","env");C(s)||C(u.join(process.cwd(),".env.local"))}c(T,"ensureEnv");function C(s){try{let t=p.readFileSync(s,"utf-8");for(let n of t.split(`
 `)){let e=n.trim();if(!e||e.startsWith("#"))continue;let o=e.indexOf("=");if(o<0)continue;let r=e.slice(0,o).trim(),i=e.slice(o+1).trim();process.env[r]||(process.env[r]=i)}return!!(process.env.RULEMETRIC_API_KEY||process.env.RULEMETRIC_ACCESS_TOKEN)}catch{return!1}}c(C,"loadEnvFile");T();async function I(s){let t={totalSessions:0,totalCalls:0,totalInputTokens:0,totalOutputTokens:0,totalCacheReadTokens:0,totalCacheWriteTokens:0,totalCostUSD:0,providers:{}},n=await m(s);t.providers={};for(let a of n)t.providers[a.provider]=(t.providers[a.provider]||0)+1;let e=new E,o=n.filter(a=>a.provider==="claude");for(let a of o)try{let h=p.readdirSync(a.path).filter(f=>f.endsWith(".jsonl"));t.totalSessions+=h.length;for(let f of h){let v=p.readFileSync(u.join(a.path,f),"utf-8"),R=e.parse(v);for(let l of R.events)l.tokenUsage&&(t.totalCalls++,t.totalInputTokens+=l.tokenUsage.input,t.totalOutputTokens+=l.tokenUsage.output,t.totalCacheReadTokens+=l.tokenUsage.cacheRead??0,t.totalCacheWriteTokens+=l.tokenUsage.cacheWrite??0)}}catch{}let r=15,i=75;return t.totalCostUSD=Math.round((t.totalInputTokens/1e6*r+t.totalOutputTokens/1e6*i+t.totalCacheReadTokens/1e6*(r*.1)+t.totalCacheWriteTokens/1e6*(r*1.25))*100)/100,t}c(I,"loadLocalStats");async function U(s){let t=process.env.RULEMETRIC_API_URL??d,n=process.env.RULEMETRIC_API_KEY,e=process.env.RULEMETRIC_ACCESS_TOKEN,o=n??e;if(!o)return null;let r=new URLSearchParams;s&&r.set("projectPath",s);try{let i=await fetch(`${t}/api/insights/compute?${r}`,{headers:{Authorization:`Bearer ${o}`},signal:AbortSignal.timeout(5e3)});return i.ok?await i.json():null}catch{return null}}c(U,"loadApiInsights");async function g(s){let t=process.env.RULEMETRIC_API_URL??d,n=process.env.RULEMETRIC_API_KEY??process.env.RULEMETRIC_ACCESS_TOKEN;if(!n)return null;let e=new URLSearchParams;s&&e.set("projectPath",s);try{let o=await fetch(`${t}/api/insights/linked-instructions?${e}`,{headers:{Authorization:`Bearer ${n}`},signal:AbortSignal.timeout(5e3)});return o.ok?await o.json():null}catch{return null}}c(g,"loadLinkedInstructions");export{I as a,U as b,g as c};

package/dist/{chunk-GTOEWPIM.js → chunk-YQ56FQEO.js}

@@ -1,4 +1,4 @@
-import{b as u}from"./chunk-WQ2BJQYB.js";import{a as o}from"./chunk-ZLLER3HP.js";var l=`You are a meta-judge evaluating the quality of an eval run \u2014 not the code output, but whether the test cases and grading themselves are rigorous and meaningful.
+import{c as u}from"./chunk-HMMO27U4.js";import{a as o}from"./chunk-ZLLER3HP.js";var l=`You are a meta-judge evaluating the quality of an eval run \u2014 not the code output, but whether the test cases and grading themselves are rigorous and meaningful.
 
 You will receive a summary of eval cases, their expectations, and grading results.
 

package/dist/chunk-Z72VHGV7.js

@@ -0,0 +1 @@
+import{a as e}from"./chunk-ZLLER3HP.js";import{execSync as c}from"node:child_process";import{mkdtempSync as s,rmSync as f,existsSync as m}from"node:fs";import{join as d}from"node:path";import{tmpdir as l}from"node:os";function h(t){let i=t??process.cwd(),n=d(l(),"rulemetric-eval-");try{c("git rev-parse --git-dir",{cwd:i,stdio:"pipe"})}catch{let r=s(n);return{workDir:r,cleanup:e(()=>p(r),"cleanup"),isWorktree:!1}}let u;try{u=c("git rev-parse HEAD",{cwd:i,stdio:"pipe"}).toString().trim()}catch{let r=s(n);return{workDir:r,cleanup:e(()=>p(r),"cleanup"),isWorktree:!1}}let o=s(n);f(o,{recursive:!0});let a=!1;for(let r=0;r<3&&!a;r++)try{c(`git worktree add --detach "${o}" ${u}`,{cwd:i,stdio:"pipe"}),a=!0}catch{r<2&&k(150*(r+1))}if(!a){let r=s(n);return{workDir:r,cleanup:e(()=>p(r),"cleanup"),isWorktree:!1}}return{workDir:o,cleanup:e(()=>{try{c(`git worktree remove --force "${o}"`,{cwd:i,stdio:"pipe"})}catch{p(o);try{c("git worktree prune",{cwd:i,stdio:"pipe"})}catch{}}},"cleanup"),isWorktree:!0}}e(h,"createEvalWorktree");function k(t){Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,t)}e(k,"sleepSync");function p(t){try{m(t)&&f(t,{recursive:!0,force:!0})}catch{}}e(p,"safeRm");export{h as a};

package/dist/chunk-ZW5TAV3L.js

@@ -0,0 +1 @@
+import{e as $}from"./chunk-L2CFSMBC.js";import{a as v}from"./chunk-KAWNGQS2.js";import{c as f}from"./chunk-HMMO27U4.js";import{f as p}from"./chunk-H2ORWIEY.js";import{a as m}from"./chunk-ZLLER3HP.js";async function x(c,e,s=()=>{}){let{eval_:r,configuration:i,targetContent:g,runNumber:o}=c;s(`[${i}] ${r.name} run#${o} \u2014 executing...`);let n=$({targetType:e.targetType,targetContent:g,targetName:e.targetName}),d={methodology:n.methodology,delivery:n.delivery,isolated:n.isWorktree,...n.leakedFiles.length?{leakedFiles:n.leakedFiles}:{}};n.isWorktree||s(`[${i}] ${r.name} run#${o} \u2014 \u26A0 no git worktree; running without repo context`),n.leakedFiles.length&&s(`[${i}] ${r.name} run#${o} \u2014 \u26A0 CONTAMINATED: ${n.leakedFiles.join(", ")} survived the strip`);try{let a=await f({prompt:r.prompt,targetContent:n.systemPrompt,workDir:n.workDir,timeout:e.timeout,model:e.model});if(a.error)return s(`[${i}] ${r.name} run#${o} \u2014 FAILED: ${a.error}`),{evalId:r.id,evalTargetId:e.targetId,evalTargetVersion:e.targetVersion,configuration:i,runNumber:o,iteration:e.iteration,status:"failed",outputs:{error:a.error,...d},timing:{duration_ms:a.durationMs,total_tokens:a.totalTokens},grading:{}};s(`[${i}] ${r.name} run#${o} \u2014 completed in ${(a.durationMs/1e3).toFixed(1)}s`);let u={};if(!e.skipGrading&&r.expectations.length>0){s(`[${i}] ${r.name} run#${o} \u2014 grading...`);let t=await v({prompt:r.prompt,output:a.output,expectations:r.expectations,model:e.graderModel??e.model,timeout:e.timeout});u=t,s(`[${i}] ${r.name} run#${o} \u2014 grading done: pass_rate=${t.summary.pass_rate==null?"n/a (all assertions errored)":`${(t.summary.pass_rate*100).toFixed(0)}%`}`)}return{evalId:r.id,evalTargetId:e.targetId,evalTargetVersion:e.targetVersion,configuration:i,runNumber:o,iteration:e.iteration,status:"completed",outputs:{transcript:a.output,...d},timing:{duration_ms:a.durationMs,total_tokens:a.totalTokens,total_duration_seconds:a.durationMs/1e3},grading:u}}finally{n.cleanup()}}m(x,"executeAndGradeCase");async function _(c){let{cases:e,targetContent:s,concurrency:r,runNumber:i,opts:g,skipBaseline:o,onLog:n=m(()=>{},"onLog")}=c,d=o?["with_target"]:["with_target","without_target"],a=[];for(let t of e)for(let l of d)a.push({eval_:t,configuration:l,targetContent:l==="with_target"?s:void 0,runNumber:i});let u=[];for(let t=0;t<a.length;t+=Math.max(1,r)){let l=a.slice(t,t+Math.max(1,r)),k=await Promise.all(l.map(b=>x(b,g,n)));u.push(...k)}for(let t of u)try{await p("/api/eval-runs",t)}catch(l){n(`  \u26A0 failed to post a run result: ${l instanceof Error?l.message:String(l)}`)}try{await p(`/api/eval-targets/${g.targetId}/label-state`,{})}catch(t){n(`  \u26A0 label-state notification skipped: ${t instanceof Error?t.message:String(t)}`)}return{completed:u.filter(t=>t.status==="completed").length,failed:u.filter(t=>t.status==="failed").length,results:u}}m(_,"runBatchAndPost");export{x as a,_ as b};

package/dist/commands/evals/agent.js

@@ -1,3 +1,3 @@
-import{a as k,b as A,c as I}from"../../chunk-47HDPK73.js";import{e as O}from"../../chunk-QYKLIRMY.js";import"../../chunk-TIOKY2IA.js";import"../../chunk-35ST2BX7.js";import"../../chunk-MYPDODRJ.js";import"../../chunk-O23Z6LTP.js";import"../../chunk-PRDYXO6Z.js";import"../../chunk-QFVGUZDO.js";import"../../chunk-KG3TW5QB.js";import"../../chunk-W2MOTOQE.js";import"../../chunk-WNAYTVCO.js";import"../../chunk-ARZJNTMV.js";import"../../chunk-6QKI756T.js";import"../../chunk-RSKQRVPJ.js";import"../../chunk-DO3KHK2X.js";import"../../chunk-UF6US7VU.js";import"../../chunk-MSTF7A7D.js";import"../../chunk-4HZWSFSY.js";import"../../chunk-LVOLDUFC.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-N7TGRFVZ.js";import{k as u}from"../../chunk-KZ65LEUH.js";import"../../chunk-GTOEWPIM.js";import"../../chunk-6V4WM73L.js";import"../../chunk-JUF4JLZQ.js";import"../../chunk-TDD7IOV2.js";import"../../chunk-7DFNBRLY.js";import"../../chunk-MODXEGCU.js";import"../../chunk-GQRCSR2A.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-CHA5SX4Y.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as b}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{f as N}from"../../chunk-H2ORWIEY.js";import{e as P,l as g}from"../../chunk-O6C2UGBY.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as te,context as re}from"@opentelemetry/api";import{NodeSDK as M}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as V}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as $,ConsoleSpanExporter as j,SimpleSpanProcessor as q}from"@opentelemetry/sdk-trace-node";import{Resource as F}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as G,ATTR_SERVICE_VERSION as U}from"@opentelemetry/semantic-conventions";function L(t){if(!t)return{};let r={};for(let n of t.split(",")){let o=n.indexOf("=");if(o===-1)continue;let i=n.slice(0,o).trim(),a=n.slice(o+1).trim();i&&(r[i]=a)}return r}e(L,"parseOtlpHeaders");var l=null;function y(){return process.env.TELEMETRY_ENABLED!=="false"}e(y,"isTelemetryEnabled");function h(t){if(!y()||l!==null)return"off";let r=new F({[G]:t.serviceName,[U]:t.serviceVersion??"0.1.0"}),n=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,o,i;if(n){let a=L(process.env.OTEL_EXPORTER_OTLP_HEADERS),d=new V({url:`${n}/v1/traces`,headers:a});i=new $(d),o="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")i=void 0,o="silent";else{let a=new j;i=new q(a),o="console"}return l=new M({resource:r,spanProcessors:i?[i]:[]}),l.start(),o}e(h,"initTelemetry");async function f(){l&&(await l.shutdown(),l=null)}e(f,"shutdownTelemetry");import{SpanKind as fe,SpanStatusCode as me}from"@opentelemetry/api";import{trace as Ee,context as Se}from"@opentelemetry/api";import*as p from"@sentry/node";var E=!1;function S(t){if(E)return!1;let r=process.env.SENTRY_DSN;return r?(p.init({dsn:r,serverName:t.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),E=!0,!0):!1}e(S,"initSentry");async function m(t){E&&await p.flush(t)}e(m,"flushSentry");import{existsSync as B,mkdirSync as H,readFileSync as w,unlinkSync as D,writeFileSync as C}from"node:fs";import{dirname as K,join as v}from"node:path";import{execFileSync as Y}from"node:child_process";var c=v(g(),"evals-agent.lock");function J(t){try{return process.kill(t,0),!0}catch(r){return r.code==="EPERM"}}e(J,"pidAlive");function z(t){if(!J(t))return!1;try{let r=Y("ps",["-p",String(t),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(r)}catch{return!0}}e(z,"isLiveAgent");function Q(){try{H(K(c),{recursive:!0})}catch{}for(let t=0;t<2;t++)try{return C(c,String(process.pid),{flag:"wx"}),{ok:!0}}catch(r){if(r.code!=="EEXIST")throw r;let n=0;try{n=Number(w(c,"utf-8").trim())}catch{}if(n&&n!==process.pid&&z(n))return{ok:!1,holder:n};try{D(c)}catch{}}try{C(c,String(process.pid))}catch{}return{ok:!0}}e(Q,"acquireAgentLock");function T(){try{B(c)&&Number(w(c,"utf-8").trim())===process.pid&&D(c)}catch{}}e(T,"releaseAgentLock");var x=class t extends b{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(t),this.requireAuth();let r=Q();if(!r.ok){this.log(`Another rulemetric evals agent is already running (pid ${r.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",S({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let o=h({serviceName:"rulemetric-agent"});o==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):o==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!P()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
+import{a as k,b as A,c as I}from"../../chunk-47HDPK73.js";import{e as O}from"../../chunk-6N6NTMOR.js";import"../../chunk-TIOKY2IA.js";import"../../chunk-M4YF7YO7.js";import"../../chunk-RSKQRVPJ.js";import"../../chunk-35ST2BX7.js";import"../../chunk-6UTJGOBA.js";import"../../chunk-N7TGRFVZ.js";import"../../chunk-OSIP6FUD.js";import"../../chunk-ULRVMH7O.js";import"../../chunk-PRDYXO6Z.js";import"../../chunk-QFVGUZDO.js";import"../../chunk-T4P2NV3P.js";import"../../chunk-BUWWFHGO.js";import"../../chunk-MDPN46EY.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-CS2OQ3GC.js";import"../../chunk-DUP232AH.js";import"../../chunk-DO3KHK2X.js";import"../../chunk-UF6US7VU.js";import"../../chunk-MSTF7A7D.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-4HZWSFSY.js";import{k as u}from"../../chunk-KZ65LEUH.js";import"../../chunk-YQ56FQEO.js";import"../../chunk-QQ2UPNBC.js";import"../../chunk-TDD7IOV2.js";import"../../chunk-5DIOZDDZ.js";import"../../chunk-MODXEGCU.js";import"../../chunk-4VZLIGMS.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-HL755A5U.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-CJRYYV76.js";import"../../chunk-ZW5TAV3L.js";import"../../chunk-L2CFSMBC.js";import"../../chunk-Z72VHGV7.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-X2ILN3KP.js";import"../../chunk-HMMO27U4.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as b}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{f as N}from"../../chunk-H2ORWIEY.js";import{e as P,l as g}from"../../chunk-O6C2UGBY.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as te,context as re}from"@opentelemetry/api";import{NodeSDK as M}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as V}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as $,ConsoleSpanExporter as j,SimpleSpanProcessor as q}from"@opentelemetry/sdk-trace-node";import{Resource as F}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as G,ATTR_SERVICE_VERSION as U}from"@opentelemetry/semantic-conventions";function L(t){if(!t)return{};let r={};for(let n of t.split(",")){let o=n.indexOf("=");if(o===-1)continue;let i=n.slice(0,o).trim(),a=n.slice(o+1).trim();i&&(r[i]=a)}return r}e(L,"parseOtlpHeaders");var l=null;function y(){return process.env.TELEMETRY_ENABLED!=="false"}e(y,"isTelemetryEnabled");function h(t){if(!y()||l!==null)return"off";let r=new F({[G]:t.serviceName,[U]:t.serviceVersion??"0.1.0"}),n=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,o,i;if(n){let a=L(process.env.OTEL_EXPORTER_OTLP_HEADERS),d=new V({url:`${n}/v1/traces`,headers:a});i=new $(d),o="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")i=void 0,o="silent";else{let a=new j;i=new q(a),o="console"}return l=new M({resource:r,spanProcessors:i?[i]:[]}),l.start(),o}e(h,"initTelemetry");async function f(){l&&(await l.shutdown(),l=null)}e(f,"shutdownTelemetry");import{SpanKind as fe,SpanStatusCode as me}from"@opentelemetry/api";import{trace as Ee,context as Se}from"@opentelemetry/api";import*as p from"@sentry/node";var E=!1;function S(t){if(E)return!1;let r=process.env.SENTRY_DSN;return r?(p.init({dsn:r,serverName:t.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),E=!0,!0):!1}e(S,"initSentry");async function m(t){E&&await p.flush(t)}e(m,"flushSentry");import{existsSync as B,mkdirSync as H,readFileSync as w,unlinkSync as D,writeFileSync as C}from"node:fs";import{dirname as K,join as v}from"node:path";import{execFileSync as Y}from"node:child_process";var c=v(g(),"evals-agent.lock");function J(t){try{return process.kill(t,0),!0}catch(r){return r.code==="EPERM"}}e(J,"pidAlive");function z(t){if(!J(t))return!1;try{let r=Y("ps",["-p",String(t),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(r)}catch{return!0}}e(z,"isLiveAgent");function Q(){try{H(K(c),{recursive:!0})}catch{}for(let t=0;t<2;t++)try{return C(c,String(process.pid),{flag:"wx"}),{ok:!0}}catch(r){if(r.code!=="EEXIST")throw r;let n=0;try{n=Number(w(c,"utf-8").trim())}catch{}if(n&&n!==process.pid&&z(n))return{ok:!1,holder:n};try{D(c)}catch{}}try{C(c,String(process.pid))}catch{}return{ok:!0}}e(Q,"acquireAgentLock");function T(){try{B(c)&&Number(w(c,"utf-8").trim())===process.pid&&D(c)}catch{}}e(T,"releaseAgentLock");var x=class t extends b{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(t),this.requireAuth();let r=Q();if(!r.ok){this.log(`Another rulemetric evals agent is already running (pid ${r.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",S({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let o=h({serviceName:"rulemetric-agent"});o==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):o==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!P()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
 `);let a=new AbortController,d=new k({providers:[A()],statePath:v(g(),"transcript-sync-state.json"),importTranscript:e(s=>N("/api/sessions/import",s),"importTranscript"),log:e(s=>this.log(`[transcript-sync] ${s}`),"log")});I(d,{signal:a.signal,log:e(s=>this.log(`[transcript-sync] ${s}`),"log")});let R=e(async s=>{this.log(`
 [${s}] shutting down agent...`),a.abort(),await Promise.allSettled([m(2e3),f(),u()]),T(),process.exit(0)},"shutdown");process.on("SIGINT",()=>{R("SIGINT")}),process.on("SIGTERM",()=>{R("SIGTERM")});let _=!1;try{await O({signal:a.signal,log:e(s=>this.log(s),"log"),version:this.config.version,readInstalledVersion:e(()=>{try{return JSON.parse(w(v(this.config.root,"package.json"),"utf-8")).version??null}catch{return null}},"readInstalledVersion"),onVersionDrift:e(()=>{_=!0},"onVersionDrift")}),_&&(this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code."),await Promise.allSettled([m(2e3),f(),u()]),T(),process.exit(1))}finally{T(),await u()}}};export{x as default};

package/dist/commands/evals/auto-run.js

@@ -1,5 +1,5 @@
-import{a as k}from"../../chunk-CJRYYV76.js";import{b as $}from"../../chunk-UHL5NOWQ.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as w}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as i}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as A,Flags as a}from"@oclif/core";var m=class C extends w{static{h(this,"EvalAutoRun")}static description="Run evals in a loop until the effect verdict is statistically powered (\xA76f auto-run-to-power)";static examples=["<%= config.bin %> evals auto-run <target-id>            # preview (cost + plan)","<%= config.bin %> evals auto-run <target-id> --yes      # run until powered / budget","<%= config.bin %> evals auto-run <target-id> --yes --max-runs 120 --batch-parallel 4"];static args={targetId:A.string({description:"Eval target ID",required:!0})};static flags={yes:a.boolean({char:"y",description:"Actually run. Without this, only previews the plan + cost."}),"max-runs":a.integer({description:"Hard ceiling on completed runs",default:200}),"batch-parallel":a.integer({description:"Concurrent executions within a batch",default:3}),timeout:a.integer({description:"Timeout per eval in ms",default:12e4}),model:a.string({description:"Model for eval execution"}),"grader-model":a.string({description:"Model for grading"}),"skip-baseline":a.boolean({description:"Skip without_target runs (no delta \u2014 not recommended)",default:!1})};async run(){let{args:y,flags:e}=await this.parse(C);this.requireAuth();let r=y.targetId,c=20,n=await i(`/api/eval-targets/${r}`),l=await i(`/api/eval-targets/${r}/evals`);l.length===0&&this.error(`No active cases on "${n.name}". Generate some (\`evals create\` / the web Generate button) or run \`evals refine-cases\` first.`);let b=l.length*(e["skip-baseline"]?1:2),f=Math.ceil(e["max-runs"]/b);if(this.log(`Auto-run "${n.name}" v${n.version} toward a powered \xB1${c}pp verdict.`),this.log(`Plan: up to ${e["max-runs"]} runs (${l.length} active case(s) \xD7 ${e["skip-baseline"]?1:2}/batch = ${b}/batch, \u2264${f} batches). Each run is a synthetic \`claude -p\` execution + grade.`),this.log('Note: this is a controlled synthetic A/B. For a DEPLOYED instruction with real usage, the Insights observational verdict is cheaper and uses real sessions \u2014 auto-run is for pre-deploy, catalog skills, or targets with no real "without" cohort.'),!e.yes){this.log(`
-Preview only. Re-run with --yes to start.`);return}let t={runsCompleted:0,maxRuns:e["max-runs"],batchesRun:0,maxBatches:f},x={targetId:r,targetVersion:n.version,iteration:1,timeout:e.timeout,model:e.model,graderModel:e["grader-model"],skipGrading:!1};for(;;){let o;try{o=await i(`/api/eval-targets/${r}/judge-calibration`)}catch(s){this.log(`
-\u25A0 stopping \u2014 could not read the power check: ${s instanceof Error?s.message:String(s)}`);break}let E={powered:o.powerCheck?.powered??!1,mde:o.powerCheck?.mde==null?1/0:o.powerCheck.mde},B=o.caseDiscrimination??{total:0,discriminating:0,nonDiscriminating:0,unpaired:0},p=k(E,B,t,{targetEffectPp:c});if(this.log(`
-${p.action==="stop"?"\u25A0":"\u25B8"} ${p.reason}`),p.action==="stop")break;let v=await i(`/api/eval-targets/${r}/evals`);if(v.length===0){this.log("\u25A0 no active cases left \u2014 stopping.");break}let{completed:g,failed:D}=await $({cases:v,targetContent:n.content,concurrency:e["batch-parallel"],runNumber:t.batchesRun+1,opts:x,skipBaseline:e["skip-baseline"],onLog:h(s=>this.log(s),"onLog")});if(t.runsCompleted+=g,t.batchesRun+=1,this.log(`Batch ${t.batchesRun}: ${g} completed, ${D} failed \u2014 ${t.runsCompleted}/${t.maxRuns} runs, ${t.batchesRun}/${t.maxBatches} batches.`),g===0){this.log("\u25A0 batch produced 0 completed runs \u2014 the executor appears broken (check the model / `claude -p`). Aborting.");break}}let d=await i(`/api/eval-targets/${r}/judge-calibration`),u=d.powerCheck?.mde,R=u==null||!Number.isFinite(u)?"unknown":`\xB1${Math.round(u*100)}pp`;this.log(`
+import{a as k}from"../../chunk-CJRYYV76.js";import{b as $}from"../../chunk-ZW5TAV3L.js";import"../../chunk-L2CFSMBC.js";import"../../chunk-Z72VHGV7.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as w}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as A,Flags as r}from"@oclif/core";var m=class y extends w{static{h(this,"EvalAutoRun")}static description="Run evals in a loop until the effect verdict is statistically powered (\xA76f auto-run-to-power)";static examples=["<%= config.bin %> evals auto-run <target-id>            # preview (cost + plan)","<%= config.bin %> evals auto-run <target-id> --yes      # run until powered / budget","<%= config.bin %> evals auto-run <target-id> --yes --max-runs 120 --batch-parallel 4"];static args={targetId:A.string({description:"Eval target ID",required:!0})};static flags={yes:r.boolean({char:"y",description:"Actually run. Without this, only previews the plan + cost."}),"max-runs":r.integer({description:"Hard ceiling on completed runs",default:200}),"batch-parallel":r.integer({description:"Concurrent executions within a batch",default:3}),timeout:r.integer({description:"Timeout per eval in ms",default:12e4}),model:r.string({description:"Model for eval execution"}),"grader-model":r.string({description:"Model for grading"}),"skip-baseline":r.boolean({description:"Skip without_target runs (no delta \u2014 not recommended)",default:!1})};async run(){let{args:C,flags:e}=await this.parse(y);this.requireAuth();let i=C.targetId,c=20,a=await n(`/api/eval-targets/${i}`),l=await n(`/api/eval-targets/${i}/evals`);l.length===0&&this.error(`No active cases on "${a.name}". Generate some (\`evals create\` / the web Generate button) or run \`evals refine-cases\` first.`);let b=l.length*(e["skip-baseline"]?1:2),f=Math.ceil(e["max-runs"]/b);if(this.log(`Auto-run "${a.name}" v${a.version} toward a powered \xB1${c}pp verdict.`),this.log(`Plan: up to ${e["max-runs"]} runs (${l.length} active case(s) \xD7 ${e["skip-baseline"]?1:2}/batch = ${b}/batch, \u2264${f} batches). Each run is a synthetic \`claude -p\` execution + grade.`),this.log('Note: this is a controlled synthetic A/B. For a DEPLOYED instruction with real usage, the Insights observational verdict is cheaper and uses real sessions \u2014 auto-run is for pre-deploy, catalog skills, or targets with no real "without" cohort.'),!e.yes){this.log(`
+Preview only. Re-run with --yes to start.`);return}let t={runsCompleted:0,maxRuns:e["max-runs"],batchesRun:0,maxBatches:f},x={targetId:i,targetVersion:a.version,iteration:1,timeout:e.timeout,model:e.model,graderModel:e["grader-model"],skipGrading:!1,targetType:a.type,targetName:a.name};for(;;){let s;try{s=await n(`/api/eval-targets/${i}/judge-calibration`)}catch(o){this.log(`
+\u25A0 stopping \u2014 could not read the power check: ${o instanceof Error?o.message:String(o)}`);break}let E={powered:s.powerCheck?.powered??!1,mde:s.powerCheck?.mde==null?1/0:s.powerCheck.mde},B=s.caseDiscrimination??{total:0,discriminating:0,nonDiscriminating:0,unpaired:0},p=k(E,B,t,{targetEffectPp:c});if(this.log(`
+${p.action==="stop"?"\u25A0":"\u25B8"} ${p.reason}`),p.action==="stop")break;let v=await n(`/api/eval-targets/${i}/evals`);if(v.length===0){this.log("\u25A0 no active cases left \u2014 stopping.");break}let{completed:g,failed:D}=await $({cases:v,targetContent:a.content,concurrency:e["batch-parallel"],runNumber:t.batchesRun+1,opts:x,skipBaseline:e["skip-baseline"],onLog:h(o=>this.log(o),"onLog")});if(t.runsCompleted+=g,t.batchesRun+=1,this.log(`Batch ${t.batchesRun}: ${g} completed, ${D} failed \u2014 ${t.runsCompleted}/${t.maxRuns} runs, ${t.batchesRun}/${t.maxBatches} batches.`),g===0){this.log("\u25A0 batch produced 0 completed runs \u2014 the executor appears broken (check the model / `claude -p`). Aborting.");break}}let d=await n(`/api/eval-targets/${i}/judge-calibration`),u=d.powerCheck?.mde,R=u==null||!Number.isFinite(u)?"unknown":`\xB1${Math.round(u*100)}pp`;this.log(`
 Done: ${t.runsCompleted} run(s) over ${t.batchesRun} batch(es). Final MDE ${R} vs \xB1${c}pp target \u2014 powered: ${d.powerCheck?.powered?"yes":"no"}.`),d.powerCheck?.powered||this.log("Review the Benchmarks/Review tab for the current estimate; raise --max-runs to keep going.")}};export{m as default};

package/dist/commands/evals/benchmark.js

@@ -1,4 +1,4 @@
-import{b as h,c as f}from"../../chunk-WXFFD4IV.js";import"../../chunk-CHA5SX4Y.js";import{a as m}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as c}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as u}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as v}from"@oclif/core";var d=class b extends m{static{u(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={iteration:v.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:o,flags:l}=await this.parse(b);this.requireAuth();let g=await c(`/api/eval-targets/${o.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${l.iteration}`);let a=await h(o.targetId,l.iteration);a.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
+import{b as h,c as f}from"../../chunk-4O7MJUPH.js";import"../../chunk-X2ILN3KP.js";import{a as m}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as c}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as u}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as v}from"@oclif/core";var d=class b extends m{static{u(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={iteration:v.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:o,flags:l}=await this.parse(b);this.requireAuth();let g=await c(`/api/eval-targets/${o.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${l.iteration}`);let a=await h(o.targetId,l.iteration);a.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
 Aggregated ${a.totalRuns} runs
 `);for(let[e,n]of Object.entries(a.runSummary)){this.log(`${e}:`);for(let[r,i]of Object.entries(n)){let s=i;this.log(`  ${r}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`)}}let t=a.verdict;if(t)if(this.log(`
 Verdict (${t.mode}, baseline=${t.baseline}):`),!t.sufficient)this.log(`  ${t.insufficientReason??"insufficient data"} \u2014 no effect claimed`);else{let e=t.delta>=0?"+":"";this.log(`  pass-rate delta: ${e}${(t.delta*100).toFixed(0)}% (n=${t.nWith} paired)`),this.log(`  95% CI: ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%`),t.pValue!=null&&this.log(`  McNemar p: ${t.pValue.toFixed(4)}`),this.log(`  significant: ${t.significant?"YES":"no"}`)}if(a.notes.length>0){this.log(`

package/dist/commands/evals/compare.js

@@ -1,4 +1,4 @@
-import{a as $,b as A}from"../../chunk-647M2I2Y.js";import{b as p}from"../../chunk-WQ2BJQYB.js";import{a as f}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as l,f as w}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as B,Flags as b}from"@oclif/core";var c=class I extends f{static{h(this,"EvalCompareCommand")}static description="Blind A/B comparison between two eval target versions";static examples=["<%= config.bin %> evals compare <target-a> <target-b>"];static args={targetA:B.string({description:"First eval target ID",required:!0}),targetB:B.string({description:"Second eval target ID",required:!0})};static flags={model:b.string({description:"Model for execution and comparison"}),timeout:b.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:g,flags:o}=await this.parse(I);this.requireAuth();let[e,r]=await Promise.all([l(`/api/eval-targets/${g.targetA}`),l(`/api/eval-targets/${g.targetB}`)]),m=await l(`/api/eval-targets/${g.targetA}/evals`);m.length===0&&this.error("No evals found for target A. Create some first."),this.log(`Comparing "${e.name}" v${e.version} vs "${r.name}" v${r.version}`),this.log(`Running ${m.length} eval(s)...
+import{a as $,b as A}from"../../chunk-647M2I2Y.js";import{c as p}from"../../chunk-HMMO27U4.js";import{a as f}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as l,f as w}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as B,Flags as b}from"@oclif/core";var c=class I extends f{static{h(this,"EvalCompareCommand")}static description="Blind A/B comparison between two eval target versions";static examples=["<%= config.bin %> evals compare <target-a> <target-b>"];static args={targetA:B.string({description:"First eval target ID",required:!0}),targetB:B.string({description:"Second eval target ID",required:!0})};static flags={model:b.string({description:"Model for execution and comparison"}),timeout:b.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:g,flags:o}=await this.parse(I);this.requireAuth();let[e,r]=await Promise.all([l(`/api/eval-targets/${g.targetA}`),l(`/api/eval-targets/${g.targetB}`)]),m=await l(`/api/eval-targets/${g.targetA}/evals`);m.length===0&&this.error("No evals found for target A. Create some first."),this.log(`Comparing "${e.name}" v${e.version} vs "${r.name}" v${r.version}`),this.log(`Running ${m.length} eval(s)...
 `);let a=Math.random()>.5,E=a?r.content:e.content,T=a?e.content:r.content,C=a?r.name:e.name,x=a?e.name:r.name,v=[];for(let t of m){this.log(`[${t.name}] Running A...`);let s=await p({prompt:t.prompt,targetContent:E,timeout:o.timeout,model:o.model});this.log(`[${t.name}] Running B...`);let d=await p({prompt:t.prompt,targetContent:T,timeout:o.timeout,model:o.model});(s.error||d.error)&&this.error(`[${t.name}] execution failed (A: ${s.error??"ok"} | B: ${d.error??"ok"}) \u2014 aborting before the blind comparison`),v.push({evalName:t.name,outputA:s.output,outputB:d.output})}this.log(`
 Running blind comparison...`);let N=v.map(t=>`
 ## Eval: ${t.evalName}

package/dist/commands/evals/optimize-description.js

@@ -1,4 +1,4 @@
-import{a as P,b as S}from"../../chunk-WDGJHNOW.js";import{b}from"../../chunk-WQ2BJQYB.js";import{a as N}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as y,f as q,g as w}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as E}from"../../chunk-ZLLER3HP.js";import{Args as Q,Flags as v}from"@oclif/core";var C=process.env.RULEMETRIC_WEB_URL??"https://rulemetric.com",_=30*6e4,R=class k extends N{static{E(this,"EvalOptimizeDescriptionCommand")}static description="Optimize eval target description for better triggering accuracy";static examples=["<%= config.bin %> evals optimize-description <target-id>","<%= config.bin %> evals optimize-description <target-id> --queries 10"];static args={targetId:Q.string({description:"Eval target ID",required:!0})};static flags={"max-iterations":v.integer({description:"Maximum optimization iterations",default:5}),model:v.string({description:"Model for trigger testing and improvement"}),queries:v.integer({description:"Number of trigger queries to generate",default:20}),"runs-per-query":v.integer({description:"Runs per query for reliability",default:3}),"poll-interval":v.integer({description:"Ms between approval polls",default:5e3}),timeout:v.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:i,flags:t}=await this.parse(k);this.requireAuth();let e=await y(`/api/eval-targets/${i.targetId}`);this.log(`Optimizing description for "${e.name}" v${e.version}`),this.log(`Current description: "${e.description}"`),this.log("Scores measure description-gated invocation choice (catalog decision protocol)."),this.log(""),this.log("Generating trigger queries...");let r=await this.generateTriggerQueries(e,t.queries,t),a=await q(`/api/eval-targets/${i.targetId}/trigger-evals`,{queries:r}),g=`${C}/eval-targets/${i.targetId}`;this.log(`
+import{a as P,b as S}from"../../chunk-H7VP4SDO.js";import{c as b}from"../../chunk-HMMO27U4.js";import{a as N}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as y,f as q,g as w}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as E}from"../../chunk-ZLLER3HP.js";import{Args as Q,Flags as v}from"@oclif/core";var C=process.env.RULEMETRIC_WEB_URL??"https://rulemetric.com",_=30*6e4,R=class k extends N{static{E(this,"EvalOptimizeDescriptionCommand")}static description="Optimize eval target description for better triggering accuracy";static examples=["<%= config.bin %> evals optimize-description <target-id>","<%= config.bin %> evals optimize-description <target-id> --queries 10"];static args={targetId:Q.string({description:"Eval target ID",required:!0})};static flags={"max-iterations":v.integer({description:"Maximum optimization iterations",default:5}),model:v.string({description:"Model for trigger testing and improvement"}),queries:v.integer({description:"Number of trigger queries to generate",default:20}),"runs-per-query":v.integer({description:"Runs per query for reliability",default:3}),"poll-interval":v.integer({description:"Ms between approval polls",default:5e3}),timeout:v.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:i,flags:t}=await this.parse(k);this.requireAuth();let e=await y(`/api/eval-targets/${i.targetId}`);this.log(`Optimizing description for "${e.name}" v${e.version}`),this.log(`Current description: "${e.description}"`),this.log("Scores measure description-gated invocation choice (catalog decision protocol)."),this.log(""),this.log("Generating trigger queries...");let r=await this.generateTriggerQueries(e,t.queries,t),a=await q(`/api/eval-targets/${i.targetId}/trigger-evals`,{queries:r}),g=`${C}/eval-targets/${i.targetId}`;this.log(`
 Generated ${r.length} trigger queries.`),this.log("Review and approve them at:"),this.log(`  ${g}`),this.log("  (Triggers tab \u2192 click to review)"),this.log(`
 Waiting for approval...`);let c=Date.now(),d=!1;for(;!d;){Date.now()-c>_&&this.error(`Timed out after ${_/6e4} minutes waiting for approval. Approve the trigger set at ${g} (Triggers tab) and re-run this command.`),await new Promise(h=>setTimeout(h,t["poll-interval"]));let p=(await y(`/api/eval-targets/${i.targetId}/trigger-evals?status=approved`)).find(h=>h.id===a.id);p&&(d=!0,r.length=0,r.push(...p.queries),this.log("Trigger queries approved!"))}let n=r.map((u,p)=>p).sort(()=>Math.random()-.5),s=Math.ceil(n.length*.6),o=n.slice(0,s),I=n.slice(s);await w(`/api/trigger-evals/${a.id}`,{trainIndices:o,testIndices:I});let x=o.map(u=>r[u]),D=I.map(u=>r[u]);this.log(`
 Split: ${x.length} train, ${D.length} test`);let m=e.description,T=m,$=0;for(let u=1;u<=t["max-iterations"];u++){this.log(`

package/dist/commands/evals/optimize.js

@@ -1,4 +1,4 @@
-import{b as P,c as C}from"../../chunk-DMLAMJLV.js";import{a as V}from"../../chunk-GQRCSR2A.js";import{a as N}from"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-CHA5SX4Y.js";import{b as F}from"../../chunk-WQ2BJQYB.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as A}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as w,f as m,g as S}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as k}from"../../chunk-ZLLER3HP.js";import{createInterface as _}from"node:readline/promises";import{Args as q,Flags as g}from"@oclif/core";var $=class O extends A{static{k(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:q.string({description:"Eval target ID",required:!0})};static flags={auto:g.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":g.integer({description:"Maximum optimization iterations",default:5}),threshold:g.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:g.string({description:"Model for eval execution"}),"analyzer-model":g.string({description:"Model for analysis/suggestions"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:n,flags:e}=await this.parse(O);this.requireAuth();let t=await w(`/api/eval-targets/${n.targetId}`),i=await w(`/api/eval-targets/${n.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let s=[...i].sort(()=>Math.random()-.5),c=Math.ceil(s.length*.6),u=s.slice(0,c),d=s.slice(c);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${u.length} train, ${d.length} test`);let h=parseFloat(e.threshold);this.log(`Config: max ${e["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),e.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let T=t.version,f=0,x=0;for(let l=1;l<=e["max-iterations"];l++){this.log(`
+import{b as P,c as C}from"../../chunk-WYF5BIQW.js";import{a as V}from"../../chunk-4VZLIGMS.js";import{a as N}from"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-X2ILN3KP.js";import{c as F}from"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as A}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as w,f as m,g as S}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as k}from"../../chunk-ZLLER3HP.js";import{createInterface as _}from"node:readline/promises";import{Args as q,Flags as g}from"@oclif/core";var $=class O extends A{static{k(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:q.string({description:"Eval target ID",required:!0})};static flags={auto:g.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":g.integer({description:"Maximum optimization iterations",default:5}),threshold:g.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:g.string({description:"Model for eval execution"}),"analyzer-model":g.string({description:"Model for analysis/suggestions"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:n,flags:e}=await this.parse(O);this.requireAuth();let t=await w(`/api/eval-targets/${n.targetId}`),i=await w(`/api/eval-targets/${n.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let s=[...i].sort(()=>Math.random()-.5),c=Math.ceil(s.length*.6),u=s.slice(0,c),d=s.slice(c);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${u.length} train, ${d.length} test`);let h=parseFloat(e.threshold);this.log(`Config: max ${e["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),e.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let T=t.version,f=0,x=0;for(let l=1;l<=e["max-iterations"];l++){this.log(`
 === Iteration ${l} ===`),this.log("Running train evals...");let b=await this.runEvalsOnTarget(t,u,e),p=this.computePassRate(b);if(this.log(`Train pass rate: ${(p*100).toFixed(1)}%`),p>=h){this.log(`Train score ${(p*100).toFixed(1)}% >= threshold ${(h*100).toFixed(0)}%. Stopping.`);let o=await this.runEvalsOnTarget(t,d,e),z=this.computePassRate(o);this.log(`Final test score: ${(z*100).toFixed(1)}%`),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,testScore:z,analysis:{reason:"threshold_reached"},status:"completed"});break}this.log("Analyzing results...");let M=b.map(o=>({evalName:o.evalName,prompt:o.prompt,expectations:o.expectations,grading:o.grading})),a=await V({content:t.content,evalResults:M,model:e["analyzer-model"]??e.model,timeout:e.timeout});this.log(`
 Assessment: ${a.overallAssessment}`),this.log(`Suggestions: ${a.suggestions.length}`);for(let o of a.suggestions)this.log(`  - [${o.section}] ${o.reasoning}`);if(a.proposedContent===t.content){this.log("No changes proposed. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,analysis:{reason:"no_changes"},status:"completed"});break}if(!e.auto&&!await this.confirm(`
 Apply the proposed changes as a new version? [y/N] `)){this.log("Changes not applied. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,analysis:{reason:"user_declined",suggestions:a.suggestions},status:"completed"});break}let r=await S(`/api/eval-targets/${t.id}`,{content:a.proposedContent});this.log(`Created v${r.version}`),this.log(`Running test evals on v${t.version} (baseline) and v${r.version}...`);let E=await this.runEvalsOnTarget(t,d,e),R=await this.runEvalsOnTarget(r,d,e),v=this.computePassRate(R),I=this.computePassRate(E);this.log(`Test pass rate: v${t.version} ${(I*100).toFixed(1)}% \u2192 v${r.version} ${(v*100).toFixed(1)}%`);let y=P(E,R);if(y&&this.log(C(y,t.version,r.version)),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,outputVersion:r.version,trainScore:p,testScore:v,analysis:{suggestions:a.suggestions,overallAssessment:a.overallAssessment,priorTestScore:I,verdict:y},status:"completed"}),v>f)f=v,T=r.version,x=0;else if(x++,x>=2){this.log("No improvement for 2 consecutive iterations. Stopping.");break}t=r}this.log(`

package/dist/commands/evals/run.js

@@ -1,3 +1,3 @@
-import{a as $}from"../../chunk-UHL5NOWQ.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as y}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as v,f as k}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as R}from"../../chunk-ZLLER3HP.js";import{Args as I,Flags as g}from"@oclif/core";var h=class _ extends y{static{R(this,"EvalRunCommand")}static description="Run evals against a target, optionally grade results, and post to API";static examples=["<%= config.bin %> evals run <target-id>","<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2","<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"];static args={targetId:I.string({description:"Eval target ID",required:!0})};static flags={parallel:g.integer({description:"Number of parallel eval executions",default:1}),"runs-per-eval":g.integer({description:"Number of runs per eval for variance analysis",default:1}),iteration:g.integer({description:"Iteration number for tracking",default:1}),"skip-baseline":g.boolean({description:"Skip baseline (without_target) runs",default:!1}),"skip-grading":g.boolean({description:"Skip LLM grading step",default:!1}),model:g.string({description:"Model for eval execution (default: user configured)"}),"grader-model":g.string({description:"Model for grading (default: same as execution model)"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:c,flags:n}=await this.parse(_);this.requireAuth();let i=await v(`/api/eval-targets/${c.targetId}`),l=await v(`/api/eval-targets/${c.targetId}/evals`);l.length===0&&this.error(`No evals found for target "${i.name}". Create some with \`rulemetric evals create\`.`),this.log(`Running ${l.length} eval(s) for "${i.name}" v${i.version}`),this.log(`Config: ${n["runs-per-eval"]} run(s) per eval, parallel=${n.parallel}, iteration=${n.iteration}`),n["skip-baseline"]&&this.log("Skipping baseline runs"),n["skip-grading"]&&this.log("Skipping grading"),this.log("");let m=[{name:"with_target",content:i.content}];n["skip-baseline"]||m.push({name:"without_target"});let p=[];for(let r of l)for(let a of m)for(let t=1;t<=n["runs-per-eval"];t++)p.push({eval_:r,configuration:a.name,targetContent:a.content,runNumber:t});let u=[],b=n.parallel;for(let r=0;r<p.length;r+=b){let a=p.slice(r,r+b),t=await Promise.all(a.map(d=>this.executeAndGrade(d,i,n)));u.push(...t)}this.log(""),this.log("Posting results to API...");for(let r of u)await k("/api/eval-runs",r);this.log(""),this.log("=== Results Summary ===");for(let r of m){let a=u.filter(e=>e.configuration===r.name),t=a.filter(e=>e.status==="completed"),d=t.filter(e=>Object.keys(e.grading).length>0);if(this.log(`
-${r.name}:`),this.log(`  Runs: ${t.length}/${a.length} completed`),d.length>0){let e=d.map(o=>o.grading.summary?.pass_rate??0),s=e.reduce((o,f)=>o+f,0)/e.length;this.log(`  Avg pass_rate: ${(s*100).toFixed(1)}%`)}for(let e of a){let s=l.find(x=>x.id===e.evalId)?.name??e.evalId,o=e.grading.summary?.pass_rate,f=o!==void 0?` pass_rate=${(o*100).toFixed(0)}%`:"",w=e.timing.duration_ms?` ${(e.timing.duration_ms/1e3).toFixed(1)}s`:"";this.log(`  [${e.status}] ${s} run#${e.runNumber}${f}${w}`)}}if(!n["skip-baseline"]&&u.length>0){let r=u.filter(t=>t.configuration==="with_target"&&Object.keys(t.grading).length>0),a=u.filter(t=>t.configuration==="without_target"&&Object.keys(t.grading).length>0);if(r.length>0&&a.length>0){let t=r.map(s=>s.grading.summary?.pass_rate??0).reduce((s,o)=>s+o,0)/r.length,d=a.map(s=>s.grading.summary?.pass_rate??0).reduce((s,o)=>s+o,0)/a.length,e=t-d;this.log(`
-Delta: ${e>0?"+":""}${(e*100).toFixed(1)}% (with_target vs without_target)`)}}}async executeAndGrade(c,n,i){return $(c,{targetId:n.id,targetVersion:n.version,iteration:i.iteration,timeout:i.timeout,model:i.model,graderModel:i["grader-model"],skipGrading:i["skip-grading"]},l=>this.log(l))}};export{h as default};
+import{a as k}from"../../chunk-ZW5TAV3L.js";import"../../chunk-L2CFSMBC.js";import"../../chunk-Z72VHGV7.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as $}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as v,f as h}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as R}from"../../chunk-ZLLER3HP.js";import{Args as I,Flags as g}from"@oclif/core";var b=class _ extends ${static{R(this,"EvalRunCommand")}static description="Run evals against a target, optionally grade results, and post to API";static examples=["<%= config.bin %> evals run <target-id>","<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2","<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"];static args={targetId:I.string({description:"Eval target ID",required:!0})};static flags={parallel:g.integer({description:"Number of parallel eval executions",default:1}),"runs-per-eval":g.integer({description:"Number of runs per eval for variance analysis",default:1}),iteration:g.integer({description:"Iteration number for tracking",default:1}),"skip-baseline":g.boolean({description:"Skip baseline (without_target) runs",default:!1}),"skip-grading":g.boolean({description:"Skip LLM grading step",default:!1}),model:g.string({description:"Model for eval execution (default: user configured)"}),"grader-model":g.string({description:"Model for grading (default: same as execution model)"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:c,flags:r}=await this.parse(_);this.requireAuth();let i=await v(`/api/eval-targets/${c.targetId}`),l=await v(`/api/eval-targets/${c.targetId}/evals`);l.length===0&&this.error(`No evals found for target "${i.name}". Create some with \`rulemetric evals create\`.`),this.log(`Running ${l.length} eval(s) for "${i.name}" v${i.version}`),this.log(`Config: ${r["runs-per-eval"]} run(s) per eval, parallel=${r.parallel}, iteration=${r.iteration}`),r["skip-baseline"]&&this.log("Skipping baseline runs"),r["skip-grading"]&&this.log("Skipping grading"),this.log("");let m=[{name:"with_target",content:i.content}];r["skip-baseline"]||m.push({name:"without_target"});let p=[];for(let a of l)for(let n of m)for(let t=1;t<=r["runs-per-eval"];t++)p.push({eval_:a,configuration:n.name,targetContent:n.content,runNumber:t});let u=[],y=r.parallel;for(let a=0;a<p.length;a+=y){let n=p.slice(a,a+y),t=await Promise.all(n.map(d=>this.executeAndGrade(d,i,r)));u.push(...t)}this.log(""),this.log("Posting results to API...");for(let a of u)await h("/api/eval-runs",a);try{await h(`/api/eval-targets/${c.targetId}/label-state`,{})}catch{}this.log(""),this.log("=== Results Summary ===");for(let a of m){let n=u.filter(e=>e.configuration===a.name),t=n.filter(e=>e.status==="completed"),d=t.filter(e=>Object.keys(e.grading).length>0);if(this.log(`
+${a.name}:`),this.log(`  Runs: ${t.length}/${n.length} completed`),d.length>0){let e=d.map(o=>o.grading.summary?.pass_rate??0),s=e.reduce((o,f)=>o+f,0)/e.length;this.log(`  Avg pass_rate: ${(s*100).toFixed(1)}%`)}for(let e of n){let s=l.find(x=>x.id===e.evalId)?.name??e.evalId,o=e.grading.summary?.pass_rate,f=o!==void 0?` pass_rate=${(o*100).toFixed(0)}%`:"",w=e.timing.duration_ms?` ${(e.timing.duration_ms/1e3).toFixed(1)}s`:"";this.log(`  [${e.status}] ${s} run#${e.runNumber}${f}${w}`)}}if(!r["skip-baseline"]&&u.length>0){let a=u.filter(t=>t.configuration==="with_target"&&Object.keys(t.grading).length>0),n=u.filter(t=>t.configuration==="without_target"&&Object.keys(t.grading).length>0);if(a.length>0&&n.length>0){let t=a.map(s=>s.grading.summary?.pass_rate??0).reduce((s,o)=>s+o,0)/a.length,d=n.map(s=>s.grading.summary?.pass_rate??0).reduce((s,o)=>s+o,0)/n.length,e=t-d;this.log(`
+Delta: ${e>0?"+":""}${(e*100).toFixed(1)}% (with_target vs without_target)`)}}}async executeAndGrade(c,r,i){return k(c,{targetId:r.id,targetVersion:r.version,iteration:i.iteration,timeout:i.timeout,model:i.model,graderModel:i["grader-model"],skipGrading:i["skip-grading"],targetType:r.type,targetName:r.name},l=>this.log(l))}};export{b as default};

package/dist/commands/evals/validate-judge.js

@@ -1 +1 @@
-import{c,d as m}from"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as u}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as r}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as s}from"@oclif/core";var o=r(n=>n==null?"\u2014":`${(n*100).toFixed(0)}%`,"pct"),g=class n extends u{static{r(this,"EvalValidateJudge")}static description="Score a second model against your human judge-review labels (\xA76g validation pass)";static examples=["<%= config.bin %> evals validate-judge <target-id>                       # auto-detect installed CLIs","<%= config.bin %> evals validate-judge <target-id> --engine codex --model gpt-5.5","<%= config.bin %> evals validate-judge <target-id> --model claude-sonnet-5 --json","<%= config.bin %> evals validate-judge <target-id> --limit 5             # quick sample"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={model:s.string({description:"Second reviewer model. Default: resolved from your installed tooling (codex: your ~/.codex/config.toml model; claude: a tier that differs from the original judge)."}),engine:s.string({description:"CLI that runs the second reviewer. `auto` prefers your installed codex for a cross-vendor reviewer \u2014 two Claude models share a training lineage, so their agreement overstates independence.",options:["auto","claude","codex"],default:"auto"}),limit:s.integer({description:"Sample only the N most-recently-annotated runs (cost control / smoke)"}),"skip-reviewed":s.boolean({description:"Only re-grade assertions this reviewer model has not ruled on yet",default:!1}),parallel:s.integer({description:"Concurrent re-grades",default:3}),timeout:s.integer({description:"Timeout per re-grade in ms",default:12e4}),json:s.boolean({description:"Emit the full report as JSON"})};async run(){let{args:h,flags:i}=await this.parse(n);this.requireAuth();let a;try{a=await m({targetId:h.targetId,engineFlag:i.engine,modelFlag:i.model,limit:i.limit,parallel:i.parallel,timeoutMs:i.timeout,skipReviewed:i["skip-reviewed"],log:r(t=>this.log(t),"log"),warn:r(t=>this.warn(t),"warn")})}catch(t){throw t instanceof c&&this.error(t.message),t}let{report:e,judgeModels:l,reviewerModel:d}=a;if(l.length===1&&l[0]===d&&this.warn(`Second reviewer resolved to the SAME model as the original judge (${l[0]}) \u2014 this measures self-consistency, not independent review.`),i.json){this.log(JSON.stringify({target:a.targetName,judgeA:l,judgeB:d,persisted:a.persisted,alreadyReviewed:a.alreadyReviewed,...e},null,2));return}if(e.pairs===0&&a.alreadyReviewed>0){this.log(`All ${a.alreadyReviewed} labeled assertion(s) already reviewed by ${d} \u2014 nothing new to grade.`);return}if(this.log(""),this.log(`Second reviewer: ${d} \u2014 scored against ${e.pairs} human-labeled assertion(s)`+(e.skipped?` (${e.skipped} skipped)`:"")),this.log(""),this.log(`  reviewer vs HUMAN:   ${o(e.bVsHuman.overall.rate)} overall \xB7 ${o(e.bVsHuman.onJudgePass.rate)} where judge passed \xB7 ${o(e.bVsHuman.onJudgeFail.rate)} where judge failed`),this.log(`  original judge vs human: ${o(e.aVsHuman.overall.rate)} (your approve rate)`),this.log(`  reviewer vs judge:   ${o(e.bVsA.overall.rate)} (inter-model, not truth)`),this.log(`  Cohen's \u03BA (reviewer vs human): ${e.kappaBvsHuman==null?"undefined (degenerate marginals)":e.kappaBvsHuman.toFixed(2)}`),this.log(""),e.suggestedAnchorWeight!=null&&this.log(`  suggested \xA76g anchor weight if used as a passive label source: ${e.suggestedAnchorWeight.toFixed(2)} (misclassification attenuation (2\xB7acc\u22121)\xB2; validated:false either way)`),e.disagreements.length>0){this.log(""),this.log(`  ${e.disagreements.length} disagreement(s) with your labels \u2014 flagged on the Review tab:`);for(let t of e.disagreements.slice(0,10))this.log(`    run ${t.runId.slice(0,8)} #${t.expectationIndex}: human=${t.humanPassed?"pass":"fail"} reviewer=${t.judgeBPassed?"pass":"fail"} \u2014 ${t.text.slice(0,80)}`);e.disagreements.length>10&&this.log(`    \u2026 and ${e.disagreements.length-10} more (use --json for all)`)}}};export{g as default};
+import{c,d as m}from"../../chunk-HL755A5U.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as u}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as r}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as s}from"@oclif/core";var o=r(n=>n==null?"\u2014":`${(n*100).toFixed(0)}%`,"pct"),g=class n extends u{static{r(this,"EvalValidateJudge")}static description="Score a second model against your human judge-review labels (\xA76g validation pass)";static examples=["<%= config.bin %> evals validate-judge <target-id>                       # auto-detect installed CLIs","<%= config.bin %> evals validate-judge <target-id> --engine codex --model gpt-5.5","<%= config.bin %> evals validate-judge <target-id> --model claude-sonnet-5 --json","<%= config.bin %> evals validate-judge <target-id> --limit 5             # quick sample"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={model:s.string({description:"Second reviewer model. Default: resolved from your installed tooling (codex: your ~/.codex/config.toml model; claude: a tier that differs from the original judge)."}),engine:s.string({description:"CLI that runs the second reviewer. `auto` prefers your installed codex for a cross-vendor reviewer \u2014 two Claude models share a training lineage, so their agreement overstates independence.",options:["auto","claude","codex"],default:"auto"}),limit:s.integer({description:"Sample only the N most-recently-annotated runs (cost control / smoke)"}),"skip-reviewed":s.boolean({description:"Only re-grade assertions this reviewer model has not ruled on yet",default:!1}),parallel:s.integer({description:"Concurrent re-grades",default:3}),timeout:s.integer({description:"Timeout per re-grade in ms",default:12e4}),json:s.boolean({description:"Emit the full report as JSON"})};async run(){let{args:h,flags:i}=await this.parse(n);this.requireAuth();let a;try{a=await m({targetId:h.targetId,engineFlag:i.engine,modelFlag:i.model,limit:i.limit,parallel:i.parallel,timeoutMs:i.timeout,skipReviewed:i["skip-reviewed"],log:r(t=>this.log(t),"log"),warn:r(t=>this.warn(t),"warn")})}catch(t){throw t instanceof c&&this.error(t.message),t}let{report:e,judgeModels:l,reviewerModel:d}=a;if(l.length===1&&l[0]===d&&this.warn(`Second reviewer resolved to the SAME model as the original judge (${l[0]}) \u2014 this measures self-consistency, not independent review.`),i.json){this.log(JSON.stringify({target:a.targetName,judgeA:l,judgeB:d,persisted:a.persisted,alreadyReviewed:a.alreadyReviewed,...e},null,2));return}if(e.pairs===0&&a.alreadyReviewed>0){this.log(`All ${a.alreadyReviewed} labeled assertion(s) already reviewed by ${d} \u2014 nothing new to grade.`);return}if(this.log(""),this.log(`Second reviewer: ${d} \u2014 scored against ${e.pairs} human-labeled assertion(s)`+(e.skipped?` (${e.skipped} skipped)`:"")),this.log(""),this.log(`  reviewer vs HUMAN:   ${o(e.bVsHuman.overall.rate)} overall \xB7 ${o(e.bVsHuman.onJudgePass.rate)} where judge passed \xB7 ${o(e.bVsHuman.onJudgeFail.rate)} where judge failed`),this.log(`  original judge vs human: ${o(e.aVsHuman.overall.rate)} (your approve rate)`),this.log(`  reviewer vs judge:   ${o(e.bVsA.overall.rate)} (inter-model, not truth)`),this.log(`  Cohen's \u03BA (reviewer vs human): ${e.kappaBvsHuman==null?"undefined (degenerate marginals)":e.kappaBvsHuman.toFixed(2)}`),this.log(""),e.suggestedAnchorWeight!=null&&this.log(`  suggested \xA76g anchor weight if used as a passive label source: ${e.suggestedAnchorWeight.toFixed(2)} (misclassification attenuation (2\xB7acc\u22121)\xB2; validated:false either way)`),e.disagreements.length>0){this.log(""),this.log(`  ${e.disagreements.length} disagreement(s) with your labels \u2014 flagged on the Review tab:`);for(let t of e.disagreements.slice(0,10))this.log(`    run ${t.runId.slice(0,8)} #${t.expectationIndex}: human=${t.humanPassed?"pass":"fail"} reviewer=${t.judgeBPassed?"pass":"fail"} \u2014 ${t.text.slice(0,80)}`);e.disagreements.length>10&&this.log(`    \u2026 and ${e.disagreements.length-10} more (use --json for all)`)}}};export{g as default};

package/dist/dashboard/Dashboard.js

@@ -1 +1 @@
-import{a as F}from"../chunk-DVN7GFJG.js";import{a as M}from"../chunk-O2SDD2WW.js";import{a as E}from"../chunk-GBMF3TP6.js";import{a as O}from"../chunk-WSCK4WNK.js";import{a as D}from"../chunk-4Z5ECCJE.js";import"../chunk-ARNZQBMX.js";import{a as B,b as S,c as R}from"../chunk-G6XGDHGC.js";import{a as $}from"../chunk-BMGQFKG6.js";import"../chunk-RDET7CKR.js";import"../chunk-X23L2MF6.js";import"../chunk-B22HYCBV.js";import"../chunk-ZXF5NE54.js";import{a as u,b as T,d as a,e as v}from"../chunk-NJ5UED3H.js";import"../chunk-ZR3MUNLS.js";import"../chunk-7DFNBRLY.js";import{a as X}from"../chunk-3AV3GTMP.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import{a as m}from"../chunk-ZLLER3HP.js";import{useState as w,useEffect as nt,useCallback as G,useRef as st}from"react";import{render as U,Box as c,Text as o,useInput as at,useApp as lt}from"ink";import{Fragment as ht,jsx as t,jsxs as h}from"react/jsx-runtime";var dt=90;function J(){let s=process.stdout.columns||80,n=Math.min(160,s),i=n>=dt,d=i?Math.floor(n/2):n,b=Math.max(6,Math.min(10,d-34));return{dashWidth:n,wide:i,halfWidth:d,barWidth:b}}m(J,"getLayout");function z({active:s,providerName:n}){return h(c,{justifyContent:"space-between",paddingX:1,children:[t(c,{gap:1,children:u.map(i=>t(o,{bold:s===i,color:s===i?a:v,children:s===i?`[ ${T[i]} ]`:`  ${T[i]}  `},i))}),n&&t(o,{color:a,bold:!0,children:n})]})}m(z,"PeriodTabs");function H({width:s,insightsFocused:n=!1}){return t(c,{borderStyle:"round",borderColor:v,width:s,justifyContent:"center",paddingX:1,children:h(o,{children:[t(o,{color:a,bold:!0,children:"<"}),t(o,{color:a,children:">"}),t(o,{dimColor:!0,children:" switch   "}),t(o,{color:a,bold:!0,children:"q"}),t(o,{dimColor:!0,children:" quit   "}),t(o,{color:a,bold:!0,children:"s"}),h(o,{dimColor:!0,children:[" ",n?"back":"suggestions","   "]}),t(o,{color:a,bold:!0,children:"1"}),t(o,{dimColor:!0,children:"-"}),t(o,{color:a,bold:!0,children:"5"}),t(o,{dimColor:!0,children:" period   "}),t(o,{color:a,bold:!0,children:"r"}),t(o,{dimColor:!0,children:" refresh   "}),t(o,{color:a,bold:!0,children:"n"}),t(o,{dimColor:!0,children:" new session   "}),t(o,{color:a,bold:!0,children:"l"}),t(o,{dimColor:!0,children:" launcher"})]})})}m(H,"StatusBar");function ct({wide:s,width:n,children:i}){return s?t(c,{width:n,children:i}):t(ht,{children:i})}m(ct,"Row");function ut({initialLocal:s,initialApi:n,initialInstructions:i,initialPeriod:d,initialLauncherFocused:b=!1}){let{exit:j}=lt(),[e,L]=w(d),[K,Q]=w(s),[W,V]=w(n),[Z,tt]=w(i),[ot,k]=w(!1),[p,I]=w(!1),[Y,_]=w(b),et=X()?.orgSlug??null,y=st(null),it=J(),g=G(async()=>{k(!0);let[r,x,P]=await Promise.all([B(),S(),R()]);Q(r),V(x),tt(P),k(!1)},[]);nt(()=>{let r=setInterval(g,3e4);return()=>clearInterval(r)},[g]);let f=G(r=>{r!==e&&(L(r),y.current&&clearTimeout(y.current),y.current=setTimeout(g,300))},[e,g]);at((r,x)=>{if(Y)return;if(r==="q"){if(p){I(!1);return}j();return}if(r==="s"){I(A=>!A);return}if(x.escape&&p){I(!1);return}if(r==="l"&&!p){_(!0);return}if(p)return;if(r==="r"){g();return}if(r==="n"){import("../dist-FAVALJ5C.js").then(({getLauncher:A})=>{A("claude_code").launch({projectPath:process.cwd(),mode:"interactive"})});return}let P=u.indexOf(e);x.leftArrow?f(u[(P-1+u.length)%u.length]):x.rightArrow||x.tab?f(u[(P+1)%u.length]):r==="1"?f("today"):r==="2"?f("week"):r==="3"?f("30days"):r==="4"?f("month"):r==="5"&&f("all")});let{dashWidth:l,wide:q,halfWidth:rt,barWidth:C}=it,N=q?rt:l;return ot?h(c,{flexDirection:"column",width:l,children:[t(z,{active:e}),t(c,{borderStyle:"round",borderColor:a,paddingX:1,width:l,children:h(o,{dimColor:!0,children:["Loading ",T[e],"..."]})}),t(H,{width:l})]}):Y?t(c,{flexDirection:"column",width:l,children:t($,{onClose:()=>_(!1),activeOrg:et,defaultProjectPath:process.cwd(),width:l})}):h(c,{flexDirection:"column",width:l,children:[t(z,{active:e}),t(D,{local:K,api:W,label:T[e],width:l}),h(ct,{wide:q,width:l,children:[t(O,{api:W,width:N,barWidth:C}),t(F,{api:W,width:N,barWidth:C})]}),t(E,{instructions:Z,width:l,barWidth:C}),t(M,{api:W,width:l,focused:p}),t(H,{width:l,insightsFocused:p})]})}m(ut,"InteractiveDashboard");async function yt(s="week",n={}){let[i,d,b]=await Promise.all([B(),S(),R()]);if(process.stdin.isTTY&&process.stdout.isTTY){let{waitUntilExit:e}=U(t(ut,{initialLocal:i,initialApi:d,initialInstructions:b,initialPeriod:s,initialLauncherFocused:n.openLauncher??!1}));await e()}else{let e=J(),{unmount:L}=U(h(c,{flexDirection:"column",width:e.dashWidth,children:[t(D,{local:i,api:d,label:T[s],width:e.dashWidth}),t(O,{api:d,width:e.dashWidth,barWidth:e.barWidth}),t(F,{api:d,width:e.dashWidth,barWidth:e.barWidth}),t(E,{instructions:b,width:e.dashWidth,barWidth:e.barWidth}),t(M,{api:d,width:e.dashWidth})]}),{patchConsole:!1});L()}}m(yt,"renderDashboard");export{yt as renderDashboard};
+import{a as F}from"../chunk-DVN7GFJG.js";import{a as M}from"../chunk-O2SDD2WW.js";import{a as E}from"../chunk-GBMF3TP6.js";import{a as O}from"../chunk-WSCK4WNK.js";import{a as D}from"../chunk-4Z5ECCJE.js";import"../chunk-ARNZQBMX.js";import{a as B,b as S,c as R}from"../chunk-XA4KSQEC.js";import{a as $}from"../chunk-BMGQFKG6.js";import"../chunk-RDET7CKR.js";import"../chunk-X23L2MF6.js";import"../chunk-B22HYCBV.js";import"../chunk-ZXF5NE54.js";import{a as u,b as T,d as a,e as v}from"../chunk-NJ5UED3H.js";import"../chunk-ZR3MUNLS.js";import"../chunk-5DIOZDDZ.js";import{a as X}from"../chunk-3AV3GTMP.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import{a as m}from"../chunk-ZLLER3HP.js";import{useState as w,useEffect as nt,useCallback as G,useRef as st}from"react";import{render as U,Box as c,Text as o,useInput as at,useApp as lt}from"ink";import{Fragment as ht,jsx as t,jsxs as h}from"react/jsx-runtime";var dt=90;function J(){let s=process.stdout.columns||80,n=Math.min(160,s),i=n>=dt,d=i?Math.floor(n/2):n,b=Math.max(6,Math.min(10,d-34));return{dashWidth:n,wide:i,halfWidth:d,barWidth:b}}m(J,"getLayout");function z({active:s,providerName:n}){return h(c,{justifyContent:"space-between",paddingX:1,children:[t(c,{gap:1,children:u.map(i=>t(o,{bold:s===i,color:s===i?a:v,children:s===i?`[ ${T[i]} ]`:`  ${T[i]}  `},i))}),n&&t(o,{color:a,bold:!0,children:n})]})}m(z,"PeriodTabs");function H({width:s,insightsFocused:n=!1}){return t(c,{borderStyle:"round",borderColor:v,width:s,justifyContent:"center",paddingX:1,children:h(o,{children:[t(o,{color:a,bold:!0,children:"<"}),t(o,{color:a,children:">"}),t(o,{dimColor:!0,children:" switch   "}),t(o,{color:a,bold:!0,children:"q"}),t(o,{dimColor:!0,children:" quit   "}),t(o,{color:a,bold:!0,children:"s"}),h(o,{dimColor:!0,children:[" ",n?"back":"suggestions","   "]}),t(o,{color:a,bold:!0,children:"1"}),t(o,{dimColor:!0,children:"-"}),t(o,{color:a,bold:!0,children:"5"}),t(o,{dimColor:!0,children:" period   "}),t(o,{color:a,bold:!0,children:"r"}),t(o,{dimColor:!0,children:" refresh   "}),t(o,{color:a,bold:!0,children:"n"}),t(o,{dimColor:!0,children:" new session   "}),t(o,{color:a,bold:!0,children:"l"}),t(o,{dimColor:!0,children:" launcher"})]})})}m(H,"StatusBar");function ct({wide:s,width:n,children:i}){return s?t(c,{width:n,children:i}):t(ht,{children:i})}m(ct,"Row");function ut({initialLocal:s,initialApi:n,initialInstructions:i,initialPeriod:d,initialLauncherFocused:b=!1}){let{exit:j}=lt(),[e,L]=w(d),[K,Q]=w(s),[W,V]=w(n),[Z,tt]=w(i),[ot,k]=w(!1),[p,I]=w(!1),[Y,_]=w(b),et=X()?.orgSlug??null,y=st(null),it=J(),g=G(async()=>{k(!0);let[r,x,P]=await Promise.all([B(),S(),R()]);Q(r),V(x),tt(P),k(!1)},[]);nt(()=>{let r=setInterval(g,3e4);return()=>clearInterval(r)},[g]);let f=G(r=>{r!==e&&(L(r),y.current&&clearTimeout(y.current),y.current=setTimeout(g,300))},[e,g]);at((r,x)=>{if(Y)return;if(r==="q"){if(p){I(!1);return}j();return}if(r==="s"){I(A=>!A);return}if(x.escape&&p){I(!1);return}if(r==="l"&&!p){_(!0);return}if(p)return;if(r==="r"){g();return}if(r==="n"){import("../dist-BC6GMBXV.js").then(({getLauncher:A})=>{A("claude_code").launch({projectPath:process.cwd(),mode:"interactive"})});return}let P=u.indexOf(e);x.leftArrow?f(u[(P-1+u.length)%u.length]):x.rightArrow||x.tab?f(u[(P+1)%u.length]):r==="1"?f("today"):r==="2"?f("week"):r==="3"?f("30days"):r==="4"?f("month"):r==="5"&&f("all")});let{dashWidth:l,wide:q,halfWidth:rt,barWidth:C}=it,N=q?rt:l;return ot?h(c,{flexDirection:"column",width:l,children:[t(z,{active:e}),t(c,{borderStyle:"round",borderColor:a,paddingX:1,width:l,children:h(o,{dimColor:!0,children:["Loading ",T[e],"..."]})}),t(H,{width:l})]}):Y?t(c,{flexDirection:"column",width:l,children:t($,{onClose:()=>_(!1),activeOrg:et,defaultProjectPath:process.cwd(),width:l})}):h(c,{flexDirection:"column",width:l,children:[t(z,{active:e}),t(D,{local:K,api:W,label:T[e],width:l}),h(ct,{wide:q,width:l,children:[t(O,{api:W,width:N,barWidth:C}),t(F,{api:W,width:N,barWidth:C})]}),t(E,{instructions:Z,width:l,barWidth:C}),t(M,{api:W,width:l,focused:p}),t(H,{width:l,insightsFocused:p})]})}m(ut,"InteractiveDashboard");async function yt(s="week",n={}){let[i,d,b]=await Promise.all([B(),S(),R()]);if(process.stdin.isTTY&&process.stdout.isTTY){let{waitUntilExit:e}=U(t(ut,{initialLocal:i,initialApi:d,initialInstructions:b,initialPeriod:s,initialLauncherFocused:n.openLauncher??!1}));await e()}else{let e=J(),{unmount:L}=U(h(c,{flexDirection:"column",width:e.dashWidth,children:[t(D,{local:i,api:d,label:T[s],width:e.dashWidth}),t(O,{api:d,width:e.dashWidth,barWidth:e.barWidth}),t(F,{api:d,width:e.dashWidth,barWidth:e.barWidth}),t(E,{instructions:b,width:e.dashWidth,barWidth:e.barWidth}),t(M,{api:d,width:e.dashWidth})]}),{patchConsole:!1});L()}}m(yt,"renderDashboard");export{yt as renderDashboard};

package/dist/dashboard/data.js

@@ -1 +1 @@
-import{a,b,c}from"../chunk-G6XGDHGC.js";import"../chunk-7DFNBRLY.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as loadApiInsights,c as loadLinkedInstructions,a as loadLocalStats};
+import{a,b,c}from"../chunk-XA4KSQEC.js";import"../chunk-5DIOZDDZ.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as loadApiInsights,c as loadLinkedInstructions,a as loadLocalStats};

package/dist/{dist-FAVALJ5C.js → dist-BC6GMBXV.js}

@@ -1 +1 @@
-import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y}from"./chunk-7DFNBRLY.js";import"./chunk-ZLLER3HP.js";export{c as ClaudeCodeAdapter,i as CodexAdapter,f as CopilotAgentAdapter,d as CursorAdapter,g as RuleMetricMcpAdapter,b as SESSION_EVENT_TYPES,a as SESSION_TOOLS,e as VSCodeCopilotAdapter,l as buildAnalysisPrompt,u as buildTmuxAttachCommand,t as buildTmuxHasSessionArgs,r as buildTmuxLaunchArgs,s as buildTmuxSendKeysArgs,k as detectTool,o as discoverAllSessions,h as freshCodexState,j as getAdapter,n as getAllProviders,w as getAvailableLaunchers,v as getLauncher,p as getProvider,m as parseAnalysisResponse,x as sanitizeBranch,q as sanitizeTmuxName,y as worktreePathFor};
+import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y}from"./chunk-5DIOZDDZ.js";import"./chunk-ZLLER3HP.js";export{c as ClaudeCodeAdapter,i as CodexAdapter,f as CopilotAgentAdapter,d as CursorAdapter,g as RuleMetricMcpAdapter,b as SESSION_EVENT_TYPES,a as SESSION_TOOLS,e as VSCodeCopilotAdapter,l as buildAnalysisPrompt,u as buildTmuxAttachCommand,t as buildTmuxHasSessionArgs,r as buildTmuxLaunchArgs,s as buildTmuxSendKeysArgs,k as detectTool,o as discoverAllSessions,h as freshCodexState,j as getAdapter,n as getAllProviders,w as getAvailableLaunchers,v as getLauncher,p as getProvider,m as parseAnalysisResponse,x as sanitizeBranch,q as sanitizeTmuxName,y as worktreePathFor};

package/dist/lib/agent-loop.js

@@ -1 +1 @@
-import{a,b,c,d,e}from"../chunk-QYKLIRMY.js";import"../chunk-TIOKY2IA.js";import"../chunk-35ST2BX7.js";import"../chunk-MYPDODRJ.js";import"../chunk-O23Z6LTP.js";import"../chunk-PRDYXO6Z.js";import"../chunk-QFVGUZDO.js";import"../chunk-KG3TW5QB.js";import"../chunk-W2MOTOQE.js";import"../chunk-WNAYTVCO.js";import"../chunk-ARZJNTMV.js";import"../chunk-6QKI756T.js";import"../chunk-RSKQRVPJ.js";import"../chunk-DO3KHK2X.js";import"../chunk-UF6US7VU.js";import"../chunk-MSTF7A7D.js";import"../chunk-4HZWSFSY.js";import"../chunk-LVOLDUFC.js";import"../chunk-5RBLIVBF.js";import"../chunk-ZL4MZMUK.js";import"../chunk-N7TGRFVZ.js";import"../chunk-KZ65LEUH.js";import"../chunk-GTOEWPIM.js";import"../chunk-6V4WM73L.js";import"../chunk-JUF4JLZQ.js";import"../chunk-TDD7IOV2.js";import"../chunk-7DFNBRLY.js";import"../chunk-MODXEGCU.js";import"../chunk-GQRCSR2A.js";import"../chunk-UUMW6X2W.js";import"../chunk-BSXUH5FD.js";import"../chunk-6UEFCMR2.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-CHA5SX4Y.js";import"../chunk-WQ2BJQYB.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};
+import{a,b,c,d,e}from"../chunk-6N6NTMOR.js";import"../chunk-TIOKY2IA.js";import"../chunk-M4YF7YO7.js";import"../chunk-RSKQRVPJ.js";import"../chunk-35ST2BX7.js";import"../chunk-6UTJGOBA.js";import"../chunk-N7TGRFVZ.js";import"../chunk-OSIP6FUD.js";import"../chunk-ULRVMH7O.js";import"../chunk-PRDYXO6Z.js";import"../chunk-QFVGUZDO.js";import"../chunk-T4P2NV3P.js";import"../chunk-BUWWFHGO.js";import"../chunk-MDPN46EY.js";import"../chunk-5RBLIVBF.js";import"../chunk-CS2OQ3GC.js";import"../chunk-DUP232AH.js";import"../chunk-DO3KHK2X.js";import"../chunk-UF6US7VU.js";import"../chunk-MSTF7A7D.js";import"../chunk-ZL4MZMUK.js";import"../chunk-4HZWSFSY.js";import"../chunk-KZ65LEUH.js";import"../chunk-YQ56FQEO.js";import"../chunk-QQ2UPNBC.js";import"../chunk-TDD7IOV2.js";import"../chunk-5DIOZDDZ.js";import"../chunk-MODXEGCU.js";import"../chunk-4VZLIGMS.js";import"../chunk-UUMW6X2W.js";import"../chunk-HL755A5U.js";import"../chunk-6UEFCMR2.js";import"../chunk-CJRYYV76.js";import"../chunk-ZW5TAV3L.js";import"../chunk-L2CFSMBC.js";import"../chunk-Z72VHGV7.js";import"../chunk-KAWNGQS2.js";import"../chunk-CDY7TQFZ.js";import"../chunk-K25S2MGU.js";import"../chunk-X2ILN3KP.js";import"../chunk-HMMO27U4.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};

package/dist/lib/eval-analyzer.js

@@ -1 +1 @@
-import{a}from"../chunk-GQRCSR2A.js";import"../chunk-WQ2BJQYB.js";import"../chunk-ZLLER3HP.js";export{a as analyzeEvalResults};
+import{a}from"../chunk-4VZLIGMS.js";import"../chunk-HMMO27U4.js";import"../chunk-ZLLER3HP.js";export{a as analyzeEvalResults};

package/dist/lib/eval-benchmark.js

@@ -1 +1 @@
-import{a,b,c}from"../chunk-WXFFD4IV.js";import"../chunk-CHA5SX4Y.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as aggregateBenchmark,c as postBenchmark,a as summarizeRuns};
+import{a,b,c}from"../chunk-4O7MJUPH.js";import"../chunk-X2ILN3KP.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as aggregateBenchmark,c as postBenchmark,a as summarizeRuns};

package/dist/lib/eval-conversation-handler.js

@@ -1 +1 @@
-import{a}from"../chunk-JUF4JLZQ.js";import"../chunk-GQRCSR2A.js";import"../chunk-UUMW6X2W.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-WQ2BJQYB.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{a as processConversationJob};
+import{a}from"../chunk-QQ2UPNBC.js";import"../chunk-4VZLIGMS.js";import"../chunk-UUMW6X2W.js";import"../chunk-KAWNGQS2.js";import"../chunk-CDY7TQFZ.js";import"../chunk-K25S2MGU.js";import"../chunk-HMMO27U4.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{a as processConversationJob};

package/dist/lib/eval-deterministic.js

@@ -1 +1 @@
-import{a,b}from"../chunk-S6B2R2LL.js";import"../chunk-ZLLER3HP.js";export{a as parseDeterministicExpectation,b as runDeterministicCheck};
+import{a,b}from"../chunk-CDY7TQFZ.js";import"../chunk-ZLLER3HP.js";export{a as parseDeterministicExpectation,b as runDeterministicCheck};

package/dist/lib/eval-executor.js

@@ -1 +1 @@
-import{a,b}from"../chunk-WQ2BJQYB.js";import"../chunk-ZLLER3HP.js";export{b as executeEval,a as parseClaudeJsonOutput};
+import{a,b,c}from"../chunk-HMMO27U4.js";import"../chunk-ZLLER3HP.js";export{b as buildClaudeArgs,c as executeEval,a as parseClaudeJsonOutput};

package/dist/lib/eval-grader.js

@@ -1 +1 @@
-import{a}from"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-WQ2BJQYB.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-ZLLER3HP.js";export{a as gradeEvalOutput};
+import{a}from"../chunk-KAWNGQS2.js";import"../chunk-CDY7TQFZ.js";import"../chunk-K25S2MGU.js";import"../chunk-HMMO27U4.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-ZLLER3HP.js";export{a as gradeEvalOutput};

package/dist/lib/eval-meta-judge.js

@@ -1 +1 @@
-import{a}from"../chunk-GTOEWPIM.js";import"../chunk-WQ2BJQYB.js";import"../chunk-ZLLER3HP.js";export{a as judgeEvalQuality};
+import{a}from"../chunk-YQ56FQEO.js";import"../chunk-HMMO27U4.js";import"../chunk-ZLLER3HP.js";export{a as judgeEvalQuality};

package/dist/lib/eval-run-batch.js

@@ -1 +1 @@
-import{a,b}from"../chunk-UHL5NOWQ.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-WQ2BJQYB.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{a as executeAndGradeCase,b as runBatchAndPost};
+import{a,b}from"../chunk-ZW5TAV3L.js";import"../chunk-L2CFSMBC.js";import"../chunk-Z72VHGV7.js";import"../chunk-KAWNGQS2.js";import"../chunk-CDY7TQFZ.js";import"../chunk-K25S2MGU.js";import"../chunk-HMMO27U4.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{a as executeAndGradeCase,b as runBatchAndPost};

package/dist/lib/eval-trigger-tester.js

@@ -1 +1 @@
-import{a,b}from"../chunk-WDGJHNOW.js";import"../chunk-WQ2BJQYB.js";import"../chunk-ZLLER3HP.js";export{b as computeTriggerScore,a as testTrigger};
+import{a,b}from"../chunk-H7VP4SDO.js";import"../chunk-HMMO27U4.js";import"../chunk-ZLLER3HP.js";export{b as computeTriggerScore,a as testTrigger};

package/dist/lib/eval-workspace.js

@@ -0,0 +1 @@
+import{a,b,c,d,e}from"../chunk-L2CFSMBC.js";import"../chunk-Z72VHGV7.js";import"../chunk-ZLLER3HP.js";export{b as AMBIENT_INSTRUCTION_FILENAMES,a as EVAL_METHODOLOGY,e as createEvalWorkspace,c as findAmbientInstructionFiles,d as skillSlug};

package/dist/lib/eval-worktree.js

@@ -1 +1 @@
-import{a}from"../chunk-6V4WM73L.js";import"../chunk-ZLLER3HP.js";export{a as createEvalWorktree};
+import{a}from"../chunk-Z72VHGV7.js";import"../chunk-ZLLER3HP.js";export{a as createEvalWorktree};

package/dist/lib/handlers/cron-eval-autorun.js

@@ -0,0 +1 @@
+import{a}from"../../chunk-DUP232AH.js";import"../../chunk-CJRYYV76.js";import"../../chunk-ZW5TAV3L.js";import"../../chunk-L2CFSMBC.js";import"../../chunk-Z72VHGV7.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as cronEvalAutorun};

package/dist/lib/handlers/process-announcement.js

@@ -1 +1 @@
-import{a}from"../../chunk-LVOLDUFC.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-N7TGRFVZ.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-ZLLER3HP.js";export{a as processAnnouncement};
+import{a}from"../../chunk-6UTJGOBA.js";import"../../chunk-N7TGRFVZ.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-ZLLER3HP.js";export{a as processAnnouncement};

package/dist/lib/handlers/process-changelog.js

@@ -1 +1 @@
-import{a}from"../../chunk-2TX7TN5R.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-N7TGRFVZ.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-ZLLER3HP.js";export{a as processChangelog};
+import{a}from"../../chunk-CBP7QZAN.js";import"../../chunk-N7TGRFVZ.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-ZLLER3HP.js";export{a as processChangelog};

package/dist/lib/handlers/process-conversation.js

@@ -1 +1 @@
-import{a}from"../../chunk-MYPDODRJ.js";import"../../chunk-JUF4JLZQ.js";import"../../chunk-GQRCSR2A.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processConversation};
+import{a}from"../../chunk-OSIP6FUD.js";import"../../chunk-QQ2UPNBC.js";import"../../chunk-4VZLIGMS.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processConversation};

package/dist/lib/handlers/process-eval.js

@@ -1 +1 @@
-import{a}from"../../chunk-O23Z6LTP.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-GTOEWPIM.js";import"../../chunk-6V4WM73L.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processEval};
+import{a}from"../../chunk-ULRVMH7O.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-YQ56FQEO.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-L2CFSMBC.js";import"../../chunk-Z72VHGV7.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processEval};

package/dist/lib/handlers/process-judge-review.js

@@ -1 +1 @@
-import{a}from"../../chunk-KG3TW5QB.js";import"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processJudgeReview};
+import{a}from"../../chunk-T4P2NV3P.js";import"../../chunk-HL755A5U.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-HMMO27U4.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processJudgeReview};

package/dist/lib/handlers/process-launch.js

@@ -1 +1 @@
-import{a,b}from"../../chunk-W2MOTOQE.js";import"../../chunk-6QKI756T.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-7DFNBRLY.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{b as processLaunch,a as resolveWorktreeForLaunch};
+import{a,b}from"../../chunk-BUWWFHGO.js";import"../../chunk-MDPN46EY.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-5DIOZDDZ.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{b as processLaunch,a as resolveWorktreeForLaunch};

package/dist/lib/handlers/process-memory-ab.js

@@ -1 +1 @@
-import{a}from"../../chunk-WNAYTVCO.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-CHA5SX4Y.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-ZLLER3HP.js";export{a as processMemoryAb};
+import{a}from"../../chunk-CS2OQ3GC.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-X2ILN3KP.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-ZLLER3HP.js";export{a as processMemoryAb};

package/dist/lib/handlers/process-run-cleanup.js

@@ -1 +1 @@
-import{a}from"../../chunk-ARZJNTMV.js";import"../../chunk-6QKI756T.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-7DFNBRLY.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-ZLLER3HP.js";export{a as processRunCleanup};
+import{a}from"../../chunk-M4YF7YO7.js";import"../../chunk-MDPN46EY.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-5DIOZDDZ.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-ZLLER3HP.js";export{a as processRunCleanup};

package/dist/lib/judge-review-runner.js

@@ -1 +1 @@
-import{a,b,c,d}from"../chunk-BSXUH5FD.js";import"../chunk-6UEFCMR2.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-WQ2BJQYB.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{c as JudgeReviewSkip,b as llmExpectationIndexes,a as reviewMatchesReviewer,d as runJudgeReviewPass};
+import{a,b,c,d}from"../chunk-HL755A5U.js";import"../chunk-6UEFCMR2.js";import"../chunk-KAWNGQS2.js";import"../chunk-CDY7TQFZ.js";import"../chunk-K25S2MGU.js";import"../chunk-HMMO27U4.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{c as JudgeReviewSkip,b as llmExpectationIndexes,a as reviewMatchesReviewer,d as runJudgeReviewPass};

package/dist/lib/manual-tasks.js

@@ -1 +1 @@
-import{a as l}from"../chunk-2TX7TN5R.js";import{a as u}from"../chunk-UF6US7VU.js";import{a as g}from"../chunk-MSTF7A7D.js";import"../chunk-ZL4MZMUK.js";import"../chunk-N7TGRFVZ.js";import"../chunk-KZ65LEUH.js";import"../chunk-TDD7IOV2.js";import"../chunk-MODXEGCU.js";import{a as i}from"../chunk-GY5IJ4RS.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import{a as o}from"../chunk-ZLLER3HP.js";var c={cron_check_changelog:o(async(r,e)=>{let s=await l({logger:{info:o(t=>e.logger.info(t),"info"),warn:o(t=>e.logger.warn(t),"warn")}});e.logger.info(`[cron_check_changelog] result: created=${s.createdCount} skipped=${s.skippedCount}`)},"cron_check_changelog"),cron_suggest_instructions:o((r,e)=>g(r,e),"cron_suggest_instructions"),cron_refresh_skills:o((r,e)=>u(r,e),"cron_refresh_skills")},p=i.map(r=>{let e=c[r.name];if(!e)throw new Error(`manual-tasks: no handler registered for "${r.name}"`);return{...r,handler:e}});async function m(r){let e=p.find(n=>n.name===r);if(!e)return{name:r,durationMs:0,ok:!1,error:`Unknown task: ${r}`,logs:[]};let s=[],t={logger:{info:o(n=>s.push({level:"info",message:n}),"info"),warn:o(n=>s.push({level:"warn",message:n}),"warn"),error:o(n=>s.push({level:"error",message:n}),"error")},job:{attempts:0}},a=Date.now();try{return await e.handler({},t),{name:r,durationMs:Date.now()-a,ok:!0,logs:s}}catch(n){return{name:r,durationMs:Date.now()-a,ok:!1,error:n instanceof Error?n.message:String(n),logs:s}}}o(m,"runManualTask");export{p as MANUAL_TASKS,m as runManualTask};
+import{a as l}from"../chunk-CBP7QZAN.js";import"../chunk-N7TGRFVZ.js";import{a as u}from"../chunk-UF6US7VU.js";import{a as g}from"../chunk-MSTF7A7D.js";import"../chunk-ZL4MZMUK.js";import"../chunk-KZ65LEUH.js";import"../chunk-TDD7IOV2.js";import"../chunk-MODXEGCU.js";import{a as i}from"../chunk-GY5IJ4RS.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import{a as o}from"../chunk-ZLLER3HP.js";var c={cron_check_changelog:o(async(r,e)=>{let s=await l({logger:{info:o(t=>e.logger.info(t),"info"),warn:o(t=>e.logger.warn(t),"warn")}});e.logger.info(`[cron_check_changelog] result: created=${s.createdCount} skipped=${s.skippedCount}`)},"cron_check_changelog"),cron_suggest_instructions:o((r,e)=>g(r,e),"cron_suggest_instructions"),cron_refresh_skills:o((r,e)=>u(r,e),"cron_refresh_skills")},p=i.map(r=>{let e=c[r.name];if(!e)throw new Error(`manual-tasks: no handler registered for "${r.name}"`);return{...r,handler:e}});async function m(r){let e=p.find(n=>n.name===r);if(!e)return{name:r,durationMs:0,ok:!1,error:`Unknown task: ${r}`,logs:[]};let s=[],t={logger:{info:o(n=>s.push({level:"info",message:n}),"info"),warn:o(n=>s.push({level:"warn",message:n}),"warn"),error:o(n=>s.push({level:"error",message:n}),"error")},job:{attempts:0}},a=Date.now();try{return await e.handler({},t),{name:r,durationMs:Date.now()-a,ok:!0,logs:s}}catch(n){return{name:r,durationMs:Date.now()-a,ok:!1,error:n instanceof Error?n.message:String(n),logs:s}}}o(m,"runManualTask");export{p as MANUAL_TASKS,m as runManualTask};

package/dist/lib/optimize-compare.js

@@ -1 +1 @@
-import{a,b,c}from"../chunk-DMLAMJLV.js";import"../chunk-CHA5SX4Y.js";import"../chunk-ZLLER3HP.js";export{b as buildVersionVerdict,c as formatVersionVerdict,a as pairVersionResults};
+import{a,b,c}from"../chunk-WYF5BIQW.js";import"../chunk-X2ILN3KP.js";import"../chunk-ZLLER3HP.js";export{b as buildVersionVerdict,c as formatVersionVerdict,a as pairVersionResults};

package/dist/lib/worktree.js

@@ -1 +1 @@
-import{a as b,b as c,c as d}from"../chunk-6QKI756T.js";import{x as a}from"../chunk-7DFNBRLY.js";import"../chunk-N7ODX6PO.js";import"../chunk-ZLLER3HP.js";export{c as ensureWorktree,d as removeWorktree,b as runGit,a as sanitizeBranch};
+import{a as b,b as c,c as d}from"../chunk-MDPN46EY.js";import{x as a}from"../chunk-5DIOZDDZ.js";import"../chunk-N7ODX6PO.js";import"../chunk-ZLLER3HP.js";export{c as ensureWorktree,d as removeWorktree,b as runGit,a as sanitizeBranch};

package/oclif.manifest.json

@@ -4323,5 +4323,5 @@
       ]
     }
   },
-  "version": "0.7.27"
+  "version": "0.7.29"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulemetric/cli",
-  "version": "0.7.27",
+  "version": "0.7.29",
   "publishConfig": {
     "access": "public"
   },
@@ -74,9 +74,9 @@
     "postgres": "^3.4.0",
     "react": "^19.0.0",
     "zod": "^3.24.0",
-    "@rulemetric/hooks": "0.7.27",
-    "@rulemetric/proxy": "0.7.27",
-    "@rulemetric/skills-registry": "0.7.27"
+    "@rulemetric/hooks": "0.7.29",
+    "@rulemetric/proxy": "0.7.29",
+    "@rulemetric/skills-registry": "0.7.29"
   },
   "//deps": "Worker (`evals agent`) runtime deps: the @opentelemetry/* set, @sentry/node, and postgres are STATIC top-level imports loaded at worker startup (via bundled @rulemetric/{telemetry,db}) — they MUST be runtime deps or a clean `npm i -g` crashes with ModuleLoadError. `better-sqlite3` is optionalDependencies (below): it's a native module, lazy `require()`'d inside a try/catch (packages/session providers cursor/opencode) so a missing binary degrades gracefully — making it a hard dep would break `npm i -g` on platforms without prebuilt binaries. `resend` stays a devDep: lazy `await import('resend')` guarded by RESEND_API_KEY, which end-user workers never set (provider falls back to noop). Classification verified via import-graph scan of dist/commands/evals/agent.js + per-importer static/lazy check.",
   "optionalDependencies": {
@@ -95,13 +95,13 @@
     "typescript": "^5.9.0",
     "vitest": "^3.2.0",
     "@rulemetric/core": "0.0.1",
-    "@rulemetric/api": "0.0.1",
-    "@rulemetric/db": "0.0.1",
     "@rulemetric/effectiveness": "0.0.1",
     "@rulemetric/email": "0.0.1",
-    "@rulemetric/telemetry": "0.0.1",
+    "@rulemetric/api": "0.0.1",
+    "@rulemetric/session": "0.2.1",
     "@rulemetric/typescript-config": "0.0.0",
-    "@rulemetric/session": "0.2.1"
+    "@rulemetric/telemetry": "0.0.1",
+    "@rulemetric/db": "0.0.1"
   },
   "scripts": {
     "build": "node scripts/build.mjs",

package/dist/chunk-6V4WM73L.js

@@ -1 +0,0 @@
-import{a as e}from"./chunk-ZLLER3HP.js";import{execSync as o}from"node:child_process";import{mkdtempSync as n,rmSync as m,existsSync as u}from"node:fs";import{join as k}from"node:path";import{tmpdir as f}from"node:os";function y(c){let t=c??process.cwd(),s=k(f(),"rulemetric-eval-");try{o("git rev-parse --git-dir",{cwd:t,stdio:"pipe"})}catch{let r=n(s);return{workDir:r,cleanup:e(()=>p(r),"cleanup"),isWorktree:!1}}let a;try{a=o("git rev-parse HEAD",{cwd:t,stdio:"pipe"}).toString().trim()}catch{let r=n(s);return{workDir:r,cleanup:e(()=>p(r),"cleanup"),isWorktree:!1}}let i=n(s);m(i,{recursive:!0});try{o(`git worktree add --detach "${i}" ${a}`,{cwd:t,stdio:"pipe"})}catch{let r=n(s);return{workDir:r,cleanup:e(()=>p(r),"cleanup"),isWorktree:!1}}return{workDir:i,cleanup:e(()=>{try{o(`git worktree remove --force "${i}"`,{cwd:t,stdio:"pipe"})}catch{p(i);try{o("git worktree prune",{cwd:t,stdio:"pipe"})}catch{}}},"cleanup"),isWorktree:!0}}e(y,"createEvalWorktree");function p(c){try{u(c)&&m(c,{recursive:!0,force:!0})}catch{}}e(p,"safeRm");export{y as a};

package/dist/chunk-O23Z6LTP.js

@@ -1 +0,0 @@
-import{a as N}from"./chunk-5RBLIVBF.js";import{a as S}from"./chunk-GTOEWPIM.js";import{a as T}from"./chunk-6V4WM73L.js";import{b as G}from"./chunk-UUMW6X2W.js";import{a as C}from"./chunk-YDRZDBHG.js";import{b as x}from"./chunk-WQ2BJQYB.js";import{e as k,f as E,h as d}from"./chunk-H2ORWIEY.js";import{a as h}from"./chunk-ZLLER3HP.js";async function W(v,f){let{mirrorId:s,evalTargetId:g,config:o}=v,a=h(e=>f.logger.info(e),"log"),i=await k(`/api/eval-targets/${g}`),$=await k(`/api/eval-targets/${g}/evals`);if($.length===0&&o.generateIfEmpty){a("No eval cases on target \u2014 generating from content via local LLM...");let e=await G({content:i.content,name:i.name});if(e===null)throw new Error("Case generation requires an LLM backend (claude CLI on PATH or ANTHROPIC_API_KEY) \u2014 none available on this worker.");let t=[];for(let n of e)t.push(await E(`/api/eval-targets/${g}/evals`,n));$=t,a(`Generated ${t.length} eval case(s)`)}if($.length===0)throw new N("No evals defined for target");let J=o.runsPerEval??1,R=o.timeout??12e4,j=o.iteration??1,y=new Set;if(f.job.attempts>1)for(let t=0;;t+=100){let n=await k(`/api/eval-targets/${g}/runs?jobId=${s}&limit=100&offset=${t}`);for(let r of n)y.add(`${r.evalId}|${r.configuration}|${r.runNumber}|${r.iteration}`);if(n.length<100)break}let _=[{name:"with_target",content:i.content}];o.skipBaseline||_.push({name:"without_target"});let p=[];for(let e of $)for(let t of _)for(let n=1;n<=J;n++)y.has(`${e.id}|${t.name}|${n}|${j}`)||p.push({eval_:e,configuration:t.name,targetContent:t.content,runNumber:n});y.size>0&&a(`Resuming attempt ${f.job.attempts}: ${y.size} run(s) already persisted, ${p.length} remaining`);let l=p.length,u=0,c=Math.min(o.parallel??1,4);await d(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:l,completed:0,current:null,message:`Starting ${l} eval runs (concurrency: ${c})...`}});let b=[];async function I(e){let t=`${e.configuration} \xB7 ${e.eval_.name} \xB7 run#${e.runNumber}`;a(`[${u+1}/${l}] ${t}`);let n=c>1?T():null;try{let r=await x({prompt:e.eval_.prompt,targetContent:e.targetContent,workDir:n?.workDir,timeout:R,model:o.model}),m={};return!o.skipGrading&&e.eval_.expectations.length>0&&!r.error&&(m=await C({prompt:e.eval_.prompt,output:r.output,expectations:e.eval_.expectations,model:o.graderModel??o.model,timeout:R})),{jobId:s,evalId:e.eval_.id,evalTargetId:i.id,evalTargetVersion:i.version,configuration:e.configuration,runNumber:e.runNumber,iteration:j,status:r.error?"failed":"completed",outputs:r.error?{error:r.error}:{transcript:r.output,...r.model?{model:r.model}:{}},timing:{duration_ms:r.durationMs,total_tokens:r.totalTokens},grading:m,worktree:n?.isWorktree?!0:void 0}}finally{n?.cleanup()}}if(h(I,"processItem"),c<=1)for(let e of p){let t=`${e.configuration} \xB7 ${e.eval_.name} \xB7 run#${e.runNumber}`;await d(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:l,completed:u,current:t,message:`Executing ${t}...`}});let n=await I(e);await E("/api/eval-runs",n),b.push(n),u++,await d(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:l,completed:u,current:t,message:`Completed ${t}`}})}else{a(`Running ${l} evals with concurrency=${c} (worktree isolation)`);for(let e=0;e<p.length;e+=c){let t=p.slice(e,e+c),n=`batch ${Math.floor(e/c)+1}/${Math.ceil(p.length/c)}`;await d(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:l,completed:u,current:n,message:`Executing ${n} (${t.length} parallel)...`}});let r=await Promise.all(t.map(I));for(let m of r)await E("/api/eval-runs",m),b.push(m),u++;await d(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:l,completed:u,current:n,message:`Completed ${n}`}})}}let P=z(b),w;if(!o.skipGrading&&b.some(e=>e.grading&&Object.keys(e.grading).length>0))try{a(`Running meta-judge for job ${s}...`);let e=$.map(t=>{let r=b.filter(m=>m.evalId===t.id&&m.configuration==="with_target")[0];return{evalName:t.name,prompt:t.prompt,expectations:t.expectations,grading:r?.grading?r.grading:null,output:r?.outputs?.transcript?.slice(0,500)}});w=await S({targetName:i.name,evalResults:e,model:o.graderModel??o.model,timeout:6e4}),a(`Meta-judge: quality=${w.quality_score}/5, issues=${w.issues.length}`)}catch(e){a(`Meta-judge failed (non-fatal): ${e instanceof Error?e.message:String(e)}`)}let M=w?{...P,meta_judgement:w}:P;return a(`Job ${s} completed: ${u}/${l} runs`),await d(`/api/eval-jobs/${s}/state`,{result:M}).catch(e=>{a(`result persist failed: ${e instanceof Error?e.message:String(e)}`)}),M}h(W,"processEval");function z(v){let f=v.filter(a=>a.status==="completed"),s=v.filter(a=>a.status==="failed"),g=f.filter(a=>{let i=a.grading;return i&&Object.keys(i).length>0}),o=g.length>0?g.reduce((a,i)=>a+(i.grading?.summary?.pass_rate??0),0)/g.length:null;return{total:v.length,completed:f.length,failed:s.length,avgPassRate:o}}h(z,"buildSummary");export{W as a};

package/dist/chunk-QYKLIRMY.js

@@ -1 +0,0 @@
-import{c as N}from"./chunk-TIOKY2IA.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as S}from"./chunk-MYPDODRJ.js";import{a as h}from"./chunk-O23Z6LTP.js";import{a as I}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-KG3TW5QB.js";import{b as E}from"./chunk-W2MOTOQE.js";import{a as M}from"./chunk-WNAYTVCO.js";import{a as H}from"./chunk-ARZJNTMV.js";import{a as A}from"./chunk-RSKQRVPJ.js";import{a as x}from"./chunk-DO3KHK2X.js";import{a as k}from"./chunk-UF6US7VU.js";import{a as v}from"./chunk-MSTF7A7D.js";import{b as R}from"./chunk-4HZWSFSY.js";import{a as T}from"./chunk-LVOLDUFC.js";import{b}from"./chunk-5RBLIVBF.js";import{d as w}from"./chunk-3BVHPQZW.js";import{b as f,e as _,f as p}from"./chunk-H2ORWIEY.js";import{a as s}from"./chunk-ZLLER3HP.js";var B={process_eval:h,process_insights:I,process_launch:E,process_send_message:A,process_conversation:S,process_session_goal:y,distill_memories:R,memory_ab:M,process_announcement:T,process_run_cleanup:H,cron_suggest_instructions:v,cron_refresh_skills:k,cron_refresh_pricing:x,process_judge_review:L},P=Object.keys(B),W=new Set(["process_session_goal","process_announcement","process_run_cleanup","distill_memories","memory_ab"]);function $(e=process.env){return e.DATABASE_URL?P:P.filter(n=>!W.has(n))}s($,"availableCapabilities");var F=3e4,J=3e4,U=3e4,C=250,D=5e3,K=`thin-agent-${process.pid}`,g=null;function G(e,n,r,l){let c;try{let d=w();c={claude:d.claudeBin!=null,codex:d.codexBin!=null,codexModel:d.codexConfiguredModel}}catch{}let i=s(d=>{p("/api/workers/heartbeat",{workerType:"cli-worker",clientName:K,clientVersion:l,workerVersion:"0.1.0",currentJobId:null,inFlightCount:0,lastError:null,status:d,metadata:{capabilities:e,mode:"thin-agent",llmEngines:c}}).then(m=>{let t=m?.advisory;t&&t.message!==g?(g=t.message,a(r,{event:"worker_advisory",code:t.code,message:t.message})):t||(g=null)}).catch(m=>{a(r,{event:"presence_heartbeat_error",error:m instanceof Error?m.message:String(m)})})},"send");i("alive");let o=setInterval(()=>i("alive"),U);n.addEventListener("abort",()=>{clearInterval(o),i("shutting_down")},{once:!0})}s(G,"startPresenceHeartbeat");function a(e,n){try{e(JSON.stringify({ts:new Date().toISOString(),...n}))}catch{e(`[agent-loop] log_error event=${n.event}`)}}s(a,"structuredLog");function V(e,n){return n.aborted?Promise.resolve(!1):e<=0?Promise.resolve(!0):new Promise(r=>{let l=setTimeout(()=>{n.removeEventListener("abort",c),r(!0)},e),c=s(()=>{clearTimeout(l),n.removeEventListener("abort",c),r(!1)},"onAbort");n.addEventListener("abort",c,{once:!0})})}s(V,"abortableSleep");async function q(e,n){let r=`capabilities=${encodeURIComponent(e.join(","))}&waitMs=${n}`,l=await _(`/api/work/next?${r}`);return l==null||typeof l=="string"?null:l}s(q,"pollForWork");function Q(e,n){return{logger:{info:s(r=>n(r),"info"),warn:s(r=>n(r),"warn"),error:s(r=>n(r),"error")},job:{attempts:e}}}s(Q,"makeHelpers");async function X(e,n,r){let l=B[e.kind],c=Date.now();if(a(r,{event:"claim",jobId:e.id,kind:e.kind,attempts:e.attempts}),!l){let o=`unknown kind: ${e.kind}`;a(r,{event:"fail",jobId:e.id,kind:e.kind,terminal:!0,error:o}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:o,terminal:!0}).catch(()=>{});return}let i=setInterval(()=>{a(r,{event:"heartbeat",jobId:e.id}),p(`/api/work/${e.id}/heartbeat`,{leaseToken:e.leaseToken}).catch(o=>{a(r,{event:"heartbeat_error",jobId:e.id,error:o instanceof Error?o.message:String(o)})})},n);try{let o=Q(e.attempts,r),d=await l(e.payload,o)??{};clearInterval(i),a(r,{event:"complete",jobId:e.id,kind:e.kind,durationMs:Date.now()-c}),await p(`/api/work/${e.id}/complete`,{leaseToken:e.leaseToken,result:d})}catch(o){clearInterval(i);let d=b(o),m=o instanceof Error?o.message:String(o);a(r,{event:"fail",jobId:e.id,kind:e.kind,durationMs:Date.now()-c,terminal:d,error:m}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:m,terminal:d}).catch(t=>{a(r,{event:"fail_post_error",jobId:e.id,error:t instanceof Error?t.message:String(t)})})}}s(X,"runOne");async function fe(e){let n=e.log??(()=>{}),r=e.pollWaitMs??F,l=e.heartbeatIntervalMs??J,c=e.capabilities??$(),i=C;a(n,{event:"agent_start",capabilities:c,pollWaitMs:r,heartbeatIntervalMs:l}),G(c,e.signal,n,e.version),N({signal:e.signal,log:n}).catch(t=>{a(n,{event:"tt_heartbeat_fatal",error:t instanceof Error?t.message:String(t)})});let o=5*6e4,d=0,m=s(()=>{if(!e.readInstalledVersion||!e.version)return!1;let t=Date.now();if(t-d<o)return!1;d=t;let u=e.readInstalledVersion();return!!(u&&u!==e.version)},"versionDrifted");for(;!e.signal.aborted;){if(m()){a(n,{event:"agent_version_drift",bootVersion:e.version,installedVersion:e.readInstalledVersion?.()??null}),e.onVersionDrift?.();break}let t=null;try{t=await q(c,r)}catch(u){let O=u instanceof f?u.statusCode:void 0;if(a(n,{event:"poll_error",status:O,error:u instanceof Error?u.message:String(u)}),!await V(i,e.signal))break;i=Math.min(i*2,D);continue}if(!t){if(a(n,{event:"poll_204"}),!await V(i,e.signal))break;i=Math.min(i*2,D);continue}i=C,await X(t,l,n)}a(n,{event:"agent_stop"})}s(fe,"runAgentLoop");export{B as a,P as b,$ as c,q as d,fe as e};

package/dist/chunk-S6B2R2LL.js

@@ -1 +0,0 @@
-import{a as r}from"./chunk-ZLLER3HP.js";var a=[{prefix:"contains:",kind:"contains"},{prefix:"icontains:",kind:"icontains"},{prefix:"not-contains:",kind:"not-contains"},{prefix:"regex:",kind:"regex"}];function o(e){if(e.trim()==="json-valid")return{kind:"json-valid",arg:""};for(let{prefix:n,kind:i}of a)if(e.startsWith(n))return{kind:i,arg:e.slice(n.length).trim()};return null}r(o,"parseDeterministicExpectation");function d(e,n){switch(e.kind){case"contains":return n.includes(e.arg)?{passed:!0,evidence:`Output contains "${e.arg}"`}:{passed:!1,evidence:`"${e.arg}" not found in output`};case"icontains":return n.toLowerCase().includes(e.arg.toLowerCase())?{passed:!0,evidence:`Output contains "${e.arg}" (case-insensitive)`}:{passed:!1,evidence:`"${e.arg}" not found in output (case-insensitive)`};case"not-contains":return n.includes(e.arg)?{passed:!1,evidence:`Forbidden substring "${e.arg}" is present in output`}:{passed:!0,evidence:`"${e.arg}" is absent from output`};case"regex":{let i;try{i=new RegExp(e.arg)}catch(s){return{passed:!1,evidence:`Invalid regex "${e.arg}": ${s instanceof Error?s.message:String(s)}`}}let t=n.match(i);return t?{passed:!0,evidence:`Pattern matched: "${t[0]}"`}:{passed:!1,evidence:`Pattern /${e.arg}/ did not match output`}}case"json-valid":{let i=[n.trim()],t=n.match(/```(?:json)?\s*([\s\S]*?)```/);t&&i.push(t[1].trim());for(let s of i)try{return JSON.parse(s),{passed:!0,evidence:"Output parses as valid JSON"}}catch{}return{passed:!1,evidence:"No parseable JSON found in output (checked whole output and fenced blocks)"}}}}r(d,"runDeterministicCheck");export{o as a,d as b};

package/dist/chunk-UHL5NOWQ.js

@@ -1 +0,0 @@
-import{a as v}from"./chunk-YDRZDBHG.js";import{b as f}from"./chunk-WQ2BJQYB.js";import{f as p}from"./chunk-H2ORWIEY.js";import{a as d}from"./chunk-ZLLER3HP.js";async function $(m,t,s=()=>{}){let{eval_:e,configuration:a,targetContent:c,runNumber:i}=m;s(`[${a}] ${e.name} run#${i} \u2014 executing...`);let r=await f({prompt:e.prompt,targetContent:c,timeout:t.timeout,model:t.model});if(r.error)return s(`[${a}] ${e.name} run#${i} \u2014 FAILED: ${r.error}`),{evalId:e.id,evalTargetId:t.targetId,evalTargetVersion:t.targetVersion,configuration:a,runNumber:i,iteration:t.iteration,status:"failed",outputs:{error:r.error},timing:{duration_ms:r.durationMs,total_tokens:r.totalTokens},grading:{}};s(`[${a}] ${e.name} run#${i} \u2014 completed in ${(r.durationMs/1e3).toFixed(1)}s`);let l={};if(!t.skipGrading&&e.expectations.length>0){s(`[${a}] ${e.name} run#${i} \u2014 grading...`);let u=await v({prompt:e.prompt,output:r.output,expectations:e.expectations,model:t.graderModel??t.model,timeout:t.timeout});l=u,s(`[${a}] ${e.name} run#${i} \u2014 grading done: pass_rate=${(u.summary.pass_rate*100).toFixed(0)}%`)}return{evalId:e.id,evalTargetId:t.targetId,evalTargetVersion:t.targetVersion,configuration:a,runNumber:i,iteration:t.iteration,status:"completed",outputs:{transcript:r.output},timing:{duration_ms:r.durationMs,total_tokens:r.totalTokens,total_duration_seconds:r.durationMs/1e3},grading:l}}d($,"executeAndGradeCase");async function h(m){let{cases:t,targetContent:s,concurrency:e,runNumber:a,opts:c,skipBaseline:i,onLog:r=d(()=>{},"onLog")}=m,l=i?["with_target"]:["with_target","without_target"],u=[];for(let n of t)for(let o of l)u.push({eval_:n,configuration:o,targetContent:o==="with_target"?s:void 0,runNumber:a});let g=[];for(let n=0;n<u.length;n+=Math.max(1,e)){let o=u.slice(n,n+Math.max(1,e)),x=await Promise.all(o.map(b=>$(b,c,r)));g.push(...x)}for(let n of g)try{await p("/api/eval-runs",n)}catch(o){r(`  \u26A0 failed to post a run result: ${o instanceof Error?o.message:String(o)}`)}return{completed:g.filter(n=>n.status==="completed").length,failed:g.filter(n=>n.status==="failed").length,results:g}}d(h,"runBatchAndPost");export{$ as a,h as b};

package/dist/chunk-WNAYTVCO.js

@@ -1 +0,0 @@
-import{e as u,f as t,j as x}from"./chunk-KZ65LEUH.js";import{a as E}from"./chunk-YDRZDBHG.js";import{a as A}from"./chunk-CHA5SX4Y.js";import{b as P}from"./chunk-WQ2BJQYB.js";import{a as f}from"./chunk-ZLLER3HP.js";import{eq as s}from"drizzle-orm";var v=f(m=>m.summary.pass_rate>=1?1:0,"fullPassScore");async function q(m,d={}){let r=d.db??x(),w=d.executeEval??P,y=d.grade??E,n=d.now??Date.now,[e]=await r.select().from(t).where(s(t.id,m.experimentId)).limit(1);if(!e)return{skipped:"not_found"};if(e.status==="completed"||e.status==="failed")return{skipped:"already_terminal",status:e.status};let[g]=await r.select({body:u.body}).from(u).where(s(u.id,e.memoryId)).limit(1);if(!g)return await r.update(t).set({status:"failed",error:"memory no longer exists",completedAt:new Date(n())}).where(s(t.id,e.id)),{skipped:"memory_gone",status:"failed"};await r.update(t).set({status:"running",startedAt:new Date(n())}).where(s(t.id,e.id));let b=e.expectations??[],l=e.model??void 0,a=[],h=0;try{for(let i=0;i<e.trials;i++){let c=await w({prompt:e.taskPrompt,targetContent:g.body,model:l}),p=await w({prompt:e.taskPrompt,model:l});if(c.error||p.error){h++;continue}let[D,M]=await Promise.all([y({prompt:e.taskPrompt,output:c.output,expectations:b,model:l}),y({prompt:e.taskPrompt,output:p.output,expectations:b,model:l})]);a.push({withScore:v(D),withoutScore:v(M),withLatency:c.durationMs/1e3,withoutLatency:p.durationMs/1e3})}if(a.length===0){let i=`no trials completed (${h}/${e.trials} runs errored \u2014 is the claude CLI available?)`;return await r.update(t).set({status:"failed",error:i,completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"failed",requestedTrials:e.trials,completedTrials:0}}let o=A(a);return await r.update(t).set({status:"completed",result:{verdict:o,pairs:a,completedTrials:a.length,model:l??null},completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"completed",requestedTrials:e.trials,completedTrials:a.length,significant:o.significant,delta:o.delta}}catch(o){let i=o instanceof Error?o.message:String(o);return await r.update(t).set({status:"failed",error:i.slice(0,500),completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"failed",requestedTrials:e.trials,completedTrials:a.length}}}f(q,"processMemoryAb");export{q as a};

package/dist/chunk-WQ2BJQYB.js

@@ -1 +0,0 @@
-import{a as f}from"./chunk-ZLLER3HP.js";import{spawn as O}from"node:child_process";function y(r){try{let e=JSON.parse(r),p=e.result??e.content??r,c=e.usage?.total_tokens??(e.usage?.input_tokens??0)+(e.usage?.output_tokens??0),a;if(e.modelUsage&&typeof e.modelUsage=="object"){let n=-1;for(let[l,o]of Object.entries(e.modelUsage)){let s=typeof o?.outputTokens=="number"?o.outputTokens:0;s>n&&(n=s,a=l)}}return{output:p,totalTokens:c,model:a}}catch{return{output:r,totalTokens:0}}}f(y,"parseClaudeJsonOutput");async function h(r){let{prompt:e,targetContent:p,systemPrompt:c,workDir:a=process.cwd(),timeout:n=12e4,model:l}=r,o=["-p",e,"--output-format","json"],s=p??c;s&&o.push("--system-prompt",s),l&&o.push("--model",l);let d=Date.now();return new Promise(m=>{let u={...process.env};delete u.CLAUDECODE,delete u.HTTPS_PROXY;let b=["/opt/homebrew/bin","/usr/local/bin"],T=(u.PATH??"").split(":").filter(Boolean),k=b.filter(t=>!T.includes(t));k.length&&(u.PATH=[...T,...k].join(":"));let i=O("claude",o,{cwd:a,env:u,stdio:["ignore","pipe","pipe"]}),g="",x="";i.stdout.on("data",t=>{g+=t.toString()}),i.stderr.on("data",t=>{x+=t.toString()});let E=setTimeout(()=>{i.kill("SIGTERM"),m({output:g,exitCode:null,durationMs:Date.now()-d,totalTokens:0,error:`Timed out after ${n}ms`})},n);i.on("close",t=>{clearTimeout(E);let w=Date.now()-d,{output:P,totalTokens:C,model:D}=y(g);m({output:P,exitCode:t,durationMs:w,totalTokens:C,model:D,error:t!==0?x||`Exit code ${t}`:void 0})}),i.on("error",t=>{clearTimeout(E),m({output:"",exitCode:null,durationMs:Date.now()-d,totalTokens:0,error:`Failed to spawn claude: ${t.message}`})})})}f(h,"executeEval");export{y as a,h as b};