@rulemetric/cli 0.7.28 → 0.7.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-WXFFD4IV.js → chunk-4O7MJUPH.js} +1 -1
- package/dist/{chunk-4YUTMVCC.js → chunk-6N6NTMOR.js} +1 -1
- package/dist/{chunk-HDSOJEDM.js → chunk-CS2OQ3GC.js} +1 -1
- package/dist/{chunk-DMLAMJLV.js → chunk-WYF5BIQW.js} +1 -1
- package/dist/{chunk-CHA5SX4Y.js → chunk-X2ILN3KP.js} +1 -1
- package/dist/commands/evals/agent.js +1 -1
- package/dist/commands/evals/benchmark.js +1 -1
- package/dist/commands/evals/optimize.js +1 -1
- package/dist/lib/agent-loop.js +1 -1
- package/dist/lib/eval-benchmark.js +1 -1
- package/dist/lib/handlers/process-memory-ab.js +1 -1
- package/dist/lib/optimize-compare.js +1 -1
- package/oclif.manifest.json +1 -1
- package/package.json +7 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as $}from"./chunk-
|
|
1
|
+
import{a as $}from"./chunk-X2ILN3KP.js";import{e as w,f as k}from"./chunk-H2ORWIEY.js";import{a as u}from"./chunk-ZLLER3HP.js";function h(e){if(e.length===0)return{mean:0,stddev:0,min:0,max:0};let n=e.reduce((s,a)=>s+a,0)/e.length,o=e.reduce((s,a)=>s+(a-n)**2,0)/e.length;return{mean:Math.round(n*1e3)/1e3,stddev:Math.round(Math.sqrt(o)*1e3)/1e3,min:Math.min(...e),max:Math.max(...e)}}u(h,"computeStats");var x="with_target",E="without_target";function _(e){let n=e.grading.summary?.pass_rate;return typeof n=="number"&&n>=1}u(_,"fullPass");function b(e){return e.grading.summary?.pass_rate!=null}u(b,"hasGrading");function R(e){let n=e.timing.total_tokens;return typeof n=="number"&&n>0?n:void 0}u(R,"tokensOf");function v(e){let n=e.timing.duration_ms;return typeof n=="number"&&!isNaN(n)?n/1e3:void 0}u(v,"secondsOf");function y(e,n){let o={};for(let s of e)(o[n(s)]??=[]).push(s);return o}u(y,"groupBy");function C(e,n){let o=y(e.filter(b),i=>i.evalId),s=y(n.filter(b),i=>i.evalId),a=u((i,l)=>i.runNumber-l.runNumber,"byRun"),t=[];for(let i of Object.keys(o)){let l=o[i].sort(a),d=(s[i]??[]).sort(a);for(let r=0;r<Math.min(l.length,d.length);r++){let c={withScore:_(l[r])?1:0,withoutScore:_(d[r])?1:0},f=R(l[r]),g=R(d[r]);f!=null&&g!=null&&(c.withCost=f,c.withoutCost=g);let p=v(l[r]),m=v(d[r]);p!=null&&m!=null&&(c.withLatency=p,c.withoutLatency=m),t.push(c)}}return t}u(C,"pairRunsByEval");function T(e){let n=e.filter(r=>r.status==="completed");if(n.length===0)return{runSummary:{},verdict:null,notes:["No completed runs found"],totalRuns:0};let o=y(n,r=>r.configuration),s={},a=[];for(let[r,c]of Object.entries(o)){let f=c.filter(b).map(m=>m.grading.summary.pass_rate),g=c.map(v).filter(m=>m!=null),p=c.map(R).filter(m=>m!=null);s[r]={},f.length>0&&(s[r].pass_rate=h(f)),g.length>0&&(s[r].time_seconds=h(g)),p.length>0&&(s[r].tokens=h(p))}let t=null,i=o[x]??[],l=o[E]??[],d=C(i,l);if(d.length>0){if(t=$(d),!t.sufficient)a.push(t.insufficientReason??"Insufficient data for a significance verdict.");else if(t.significant){let r=t.delta>=0?"improves":"reduces",c=t.pValue!=null?`, McNemar p=${t.pValue.toFixed(4)}`:"";a.push(`Target ${r} the pass rate by ${Math.abs(t.delta*100).toFixed(0)}% (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}${c}).`)}else a.push(`No significant effect on pass rate (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}).`);t.costDelta!=null&&Math.round(t.costDelta)!==0&&a.push(`Token cost delta: ${t.costDelta>=0?"+":""}${Math.round(t.costDelta)} tokens/task.`),t.latencyDelta!=null&&Math.abs(t.latencyDelta)>1&&a.push(`Target ${t.latencyDelta>0?"adds":"saves"} ${Math.abs(t.latencyDelta).toFixed(1)}s/task.`)}for(let r of Object.values(s))if(r.pass_rate&&r.pass_rate.stddev>.2){a.push(`High pass-rate variance (stddev=${r.pass_rate.stddev.toFixed(2)}) \u2014 results may be unreliable.`);break}return{runSummary:s,verdict:t,notes:a,totalRuns:n.length}}u(T,"summarizeRuns");async function S(e,n){let o=n?`&iteration=${n}`:"",s=100,a=[];for(let t=0;t<1e5;t+=s){let i=await w(`/api/eval-targets/${e}/runs?limit=${s}&offset=${t}&activeCasesOnly=1${o}`);if(a.push(...i),i.length<s)break}return T(a)}u(S,"aggregateBenchmark");async function B(e,n,o){return k(`/api/eval-targets/${e}/benchmarks`,{iteration:n,runSummary:o.runSummary,notes:o.notes})}u(B,"postBenchmark");export{T as a,S as b,B as c};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{c as P}from"./chunk-TIOKY2IA.js";import{a as T}from"./chunk-M4YF7YO7.js";import{a as A}from"./chunk-RSKQRVPJ.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as M}from"./chunk-6UTJGOBA.js";import{a as S}from"./chunk-OSIP6FUD.js";import{a as h}from"./chunk-ULRVMH7O.js";import{a as E}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-T4P2NV3P.js";import{b as I}from"./chunk-BUWWFHGO.js";import{b}from"./chunk-5RBLIVBF.js";import{a as H}from"./chunk-
|
|
1
|
+
import{c as P}from"./chunk-TIOKY2IA.js";import{a as T}from"./chunk-M4YF7YO7.js";import{a as A}from"./chunk-RSKQRVPJ.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as M}from"./chunk-6UTJGOBA.js";import{a as S}from"./chunk-OSIP6FUD.js";import{a as h}from"./chunk-ULRVMH7O.js";import{a as E}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-T4P2NV3P.js";import{b as I}from"./chunk-BUWWFHGO.js";import{b}from"./chunk-5RBLIVBF.js";import{a as H}from"./chunk-CS2OQ3GC.js";import{a as N}from"./chunk-DUP232AH.js";import{a as x}from"./chunk-DO3KHK2X.js";import{a as k}from"./chunk-UF6US7VU.js";import{a as v}from"./chunk-MSTF7A7D.js";import{b as R}from"./chunk-4HZWSFSY.js";import{d as w}from"./chunk-3BVHPQZW.js";import{b as f,e as _,f as p}from"./chunk-H2ORWIEY.js";import{a as s}from"./chunk-ZLLER3HP.js";var O={process_eval:h,process_insights:E,process_launch:I,process_send_message:A,process_conversation:S,process_session_goal:y,distill_memories:R,memory_ab:H,process_announcement:M,process_run_cleanup:T,cron_suggest_instructions:v,cron_refresh_skills:k,cron_refresh_pricing:x,cron_eval_autorun:N,process_judge_review:L},C=Object.keys(O),$=new Set(["process_session_goal","process_announcement","process_run_cleanup","distill_memories","memory_ab"]);function F(e=process.env){return e.DATABASE_URL?C:C.filter(n=>!$.has(n))}s(F,"availableCapabilities");var J=3e4,U=3e4,K=3e4,D=250,V=5e3,G=`thin-agent-${process.pid}`,g=null;function q(e,n,r,l){let c;try{let d=w();c={claude:d.claudeBin!=null,codex:d.codexBin!=null,codexModel:d.codexConfiguredModel}}catch{}let i=s(d=>{p("/api/workers/heartbeat",{workerType:"cli-worker",clientName:G,clientVersion:l,workerVersion:"0.1.0",currentJobId:null,inFlightCount:0,lastError:null,status:d,metadata:{capabilities:e,mode:"thin-agent",llmEngines:c}}).then(u=>{let t=u?.advisory;t&&t.message!==g?(g=t.message,a(r,{event:"worker_advisory",code:t.code,message:t.message})):t||(g=null)}).catch(u=>{a(r,{event:"presence_heartbeat_error",error:u instanceof Error?u.message:String(u)})})},"send");i("alive");let o=setInterval(()=>i("alive"),K);n.addEventListener("abort",()=>{clearInterval(o),i("shutting_down")},{once:!0})}s(q,"startPresenceHeartbeat");function a(e,n){try{e(JSON.stringify({ts:new Date().toISOString(),...n}))}catch{e(`[agent-loop] log_error event=${n.event}`)}}s(a,"structuredLog");function B(e,n){return n.aborted?Promise.resolve(!1):e<=0?Promise.resolve(!0):new Promise(r=>{let l=setTimeout(()=>{n.removeEventListener("abort",c),r(!0)},e),c=s(()=>{clearTimeout(l),n.removeEventListener("abort",c),r(!1)},"onAbort");n.addEventListener("abort",c,{once:!0})})}s(B,"abortableSleep");async function Q(e,n){let r=`capabilities=${encodeURIComponent(e.join(","))}&waitMs=${n}`,l=await _(`/api/work/next?${r}`);return l==null||typeof l=="string"?null:l}s(Q,"pollForWork");function X(e,n){return{logger:{info:s(r=>n(r),"info"),warn:s(r=>n(r),"warn"),error:s(r=>n(r),"error")},job:{attempts:e}}}s(X,"makeHelpers");async function z(e,n,r){let l=O[e.kind],c=Date.now();if(a(r,{event:"claim",jobId:e.id,kind:e.kind,attempts:e.attempts}),!l){let o=`unknown kind: ${e.kind}`;a(r,{event:"fail",jobId:e.id,kind:e.kind,terminal:!0,error:o}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:o,terminal:!0}).catch(()=>{});return}let i=setInterval(()=>{a(r,{event:"heartbeat",jobId:e.id}),p(`/api/work/${e.id}/heartbeat`,{leaseToken:e.leaseToken}).catch(o=>{a(r,{event:"heartbeat_error",jobId:e.id,error:o instanceof Error?o.message:String(o)})})},n);try{let o=X(e.attempts,r),d=await l(e.payload,o)??{};clearInterval(i),a(r,{event:"complete",jobId:e.id,kind:e.kind,durationMs:Date.now()-c}),await p(`/api/work/${e.id}/complete`,{leaseToken:e.leaseToken,result:d})}catch(o){clearInterval(i);let d=b(o),u=o instanceof Error?o.message:String(o);a(r,{event:"fail",jobId:e.id,kind:e.kind,durationMs:Date.now()-c,terminal:d,error:u}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:u,terminal:d}).catch(t=>{a(r,{event:"fail_post_error",jobId:e.id,error:t instanceof Error?t.message:String(t)})})}}s(z,"runOne");async function ve(e){let n=e.log??(()=>{}),r=e.pollWaitMs??J,l=e.heartbeatIntervalMs??U,c=e.capabilities??F(),i=D;a(n,{event:"agent_start",capabilities:c,pollWaitMs:r,heartbeatIntervalMs:l}),q(c,e.signal,n,e.version),P({signal:e.signal,log:n}).catch(t=>{a(n,{event:"tt_heartbeat_fatal",error:t instanceof Error?t.message:String(t)})});let o=5*6e4,d=0,u=s(()=>{if(!e.readInstalledVersion||!e.version)return!1;let t=Date.now();if(t-d<o)return!1;d=t;let m=e.readInstalledVersion();return!!(m&&m!==e.version)},"versionDrifted");for(;!e.signal.aborted;){if(u()){a(n,{event:"agent_version_drift",bootVersion:e.version,installedVersion:e.readInstalledVersion?.()??null}),e.onVersionDrift?.();break}let t=null;try{t=await Q(c,r)}catch(m){let W=m instanceof f?m.statusCode:void 0;if(a(n,{event:"poll_error",status:W,error:m instanceof Error?m.message:String(m)}),!await B(i,e.signal))break;i=Math.min(i*2,V);continue}if(!t){if(a(n,{event:"poll_204"}),!await B(i,e.signal))break;i=Math.min(i*2,V);continue}i=D,await z(t,l,n)}a(n,{event:"agent_stop"})}s(ve,"runAgentLoop");export{O as a,C as b,F as c,Q as d,ve as e};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{e as u,f as t,j as x}from"./chunk-KZ65LEUH.js";import{a as E}from"./chunk-KAWNGQS2.js";import{a as A}from"./chunk-
|
|
1
|
+
import{e as u,f as t,j as x}from"./chunk-KZ65LEUH.js";import{a as E}from"./chunk-KAWNGQS2.js";import{a as A}from"./chunk-X2ILN3KP.js";import{c as P}from"./chunk-HMMO27U4.js";import{a as f}from"./chunk-ZLLER3HP.js";import{eq as s}from"drizzle-orm";var v=f(m=>(m.summary.pass_rate??0)>=1?1:0,"fullPassScore");async function q(m,d={}){let r=d.db??x(),w=d.executeEval??P,y=d.grade??E,n=d.now??Date.now,[e]=await r.select().from(t).where(s(t.id,m.experimentId)).limit(1);if(!e)return{skipped:"not_found"};if(e.status==="completed"||e.status==="failed")return{skipped:"already_terminal",status:e.status};let[g]=await r.select({body:u.body}).from(u).where(s(u.id,e.memoryId)).limit(1);if(!g)return await r.update(t).set({status:"failed",error:"memory no longer exists",completedAt:new Date(n())}).where(s(t.id,e.id)),{skipped:"memory_gone",status:"failed"};await r.update(t).set({status:"running",startedAt:new Date(n())}).where(s(t.id,e.id));let b=e.expectations??[],l=e.model??void 0,a=[],h=0;try{for(let i=0;i<e.trials;i++){let c=await w({prompt:e.taskPrompt,targetContent:g.body,model:l}),p=await w({prompt:e.taskPrompt,model:l});if(c.error||p.error){h++;continue}let[D,M]=await Promise.all([y({prompt:e.taskPrompt,output:c.output,expectations:b,model:l}),y({prompt:e.taskPrompt,output:p.output,expectations:b,model:l})]);a.push({withScore:v(D),withoutScore:v(M),withLatency:c.durationMs/1e3,withoutLatency:p.durationMs/1e3})}if(a.length===0){let i=`no trials completed (${h}/${e.trials} runs errored \u2014 is the claude CLI available?)`;return await r.update(t).set({status:"failed",error:i,completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"failed",requestedTrials:e.trials,completedTrials:0}}let o=A(a);return await r.update(t).set({status:"completed",result:{verdict:o,pairs:a,completedTrials:a.length,model:l??null},completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"completed",requestedTrials:e.trials,completedTrials:a.length,significant:o.significant,delta:o.delta}}catch(o){let i=o instanceof Error?o.message:String(o);return await r.update(t).set({status:"failed",error:i.slice(0,500),completedAt:new Date(n())}).where(s(t.id,e.id)),{status:"failed",requestedTrials:e.trials,completedTrials:a.length}}}f(q,"processMemoryAb");export{q as a};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as l}from"./chunk-
|
|
1
|
+
import{a as l}from"./chunk-X2ILN3KP.js";import{a as r}from"./chunk-ZLLER3HP.js";var u=r(n=>{let t=n.grading.summary?.pass_rate;return typeof t=="number"?t:null},"passRateOf"),a=r(n=>(u(n)??0)>=1,"fullPass");function f(n,t){let i=new Map(n.filter(e=>u(e)!=null).map(e=>[e.evalName,e])),o=[];for(let e of t){if(u(e)==null)continue;let s=i.get(e.evalName);s&&o.push({withScore:a(e)?1:0,withoutScore:a(s)?1:0})}return o}r(f,"pairVersionResults");function d(n,t){let i=f(n,t);return i.length===0?null:l(i,{baseline:"prior-version"})}r(d,"buildVersionVerdict");function m(n,t,i){let o=`v${i} vs v${t}`;if(!n.sufficient)return`${o}: ${n.insufficientReason??"insufficient data"} \u2014 no effect claimed`;let e=n.delta>=0?"+":"",s=`95% CI ${(n.ci.lower*100).toFixed(0)}\u2013${(n.ci.upper*100).toFixed(0)}%`,c=n.significant?"significant":"not significant";return`${o}: ${e}${(n.delta*100).toFixed(0)}% pass rate (${s}, n=${n.nWith} paired, ${c})`}r(m,"formatVersionVerdict");export{f as a,d as b,m as c};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as r}from"./chunk-ZLLER3HP.js";function g(e,o){let t=e,i=o,n=t+i;if(n===0)return{pValue:1,nDiscordant:0};let c=Math.min(t,i),d=Math.pow(.5,n),s=d;for(let a=1;a<=c;a++)d*=(n-a+1)/a,s+=d;return{pValue:Math.min(1,2*s),nDiscordant:n}}r(g,"mcnemarExact");function w(e,o,t=10){return{sufficient:e>=t&&o>=t,minN:t,nWith:e,nWithout:o,shortfall:{with:Math.max(0,t-e),without:Math.max(0,t-o)}}}r(w,"assessSufficiency");function R(e){let o=e>>>0;return()=>{o=o+1831565813|0;let t=Math.imul(o^o>>>15,1|o);return t=t+Math.imul(t^t>>>7,61|t)^t,((t^t>>>14)>>>0)/4294967296}}r(R,"mulberry32");function b(e,o){let t=e.length;if(t===1)return e[0];let i=(t-1)*o,n=Math.floor(i),c=Math.ceil(i);return n===c?e[n]:e[n]+(i-n)*(e[c]-e[n])}r(b,"sortedQuantile");function x(e,o={}){let t=e.length;if(t===0)return{estimate:0,lower:0,upper:0,iterations:0};let i=o.iterations??1e4,n=o.ciLevel??.95,c=R(o.seed??1),d=e.reduce((u,f)=>u+f,0)/t,s=new Array(i);for(let u=0;u<i;u++){let f=0;for(let h=0;h<t;h++)f+=e[c()*t|0];s[u]=f/t}s.sort((u,f)=>u-f);let a=(1-n)/2;return{estimate:d,lower:b(s,a),upper:b(s,1-a),iterations:i}}r(x,"pairedBootstrapCI");var M=r(e=>e===0||e===1,"isBinary");function A(e,o,t){return e.every(n=>n[o]!=null&&n[t]!=null)?e.reduce((n,c)=>n+(c[o]-c[t]),0)/e.length:null}r(A,"meanPairDelta");function
|
|
1
|
+
import{a as r}from"./chunk-ZLLER3HP.js";function g(e,o){let t=e,i=o,n=t+i;if(n===0)return{pValue:1,nDiscordant:0};let c=Math.min(t,i),d=Math.pow(.5,n),s=d;for(let a=1;a<=c;a++)d*=(n-a+1)/a,s+=d;return{pValue:Math.min(1,2*s),nDiscordant:n}}r(g,"mcnemarExact");function w(e,o,t=10){return{sufficient:e>=t&&o>=t,minN:t,nWith:e,nWithout:o,shortfall:{with:Math.max(0,t-e),without:Math.max(0,t-o)}}}r(w,"assessSufficiency");function R(e){let o=e>>>0;return()=>{o=o+1831565813|0;let t=Math.imul(o^o>>>15,1|o);return t=t+Math.imul(t^t>>>7,61|t)^t,((t^t>>>14)>>>0)/4294967296}}r(R,"mulberry32");function b(e,o){let t=e.length;if(t===1)return e[0];let i=(t-1)*o,n=Math.floor(i),c=Math.ceil(i);return n===c?e[n]:e[n]+(i-n)*(e[c]-e[n])}r(b,"sortedQuantile");function x(e,o={}){let t=e.length;if(t===0)return{estimate:0,lower:0,upper:0,iterations:0};let i=o.iterations??1e4,n=o.ciLevel??.95,c=R(o.seed??1),d=e.reduce((u,f)=>u+f,0)/t,s=new Array(i);for(let u=0;u<i;u++){let f=0;for(let h=0;h<t;h++)f+=e[c()*t|0];s[u]=f/t}s.sort((u,f)=>u-f);let a=(1-n)/2;return{estimate:d,lower:b(s,a),upper:b(s,1-a),iterations:i}}r(x,"pairedBootstrapCI");var M=r(e=>e===0||e===1,"isBinary");function A(e,o,t){return e.every(n=>n[o]!=null&&n[t]!=null)?e.reduce((n,c)=>n+(c[o]-c[t]),0)/e.length:null}r(A,"meanPairDelta");function T(e,o={}){let t=o.baseline??"no-rule",i=o.minN??10,n=e.length,c=e.map(l=>l.withScore-l.withoutScore),d=n===0?0:c.reduce((l,m)=>l+m,0)/n,s=x(c,{seed:o.seed,iterations:o.iterations}),a={lower:s.lower,upper:s.upper},u=e.every(l=>M(l.withScore)&&M(l.withoutScore)),f=null;if(u&&n>0){let l=0,m=0;for(let p of e)p.withScore===1&&p.withoutScore===0?l++:p.withScore===0&&p.withoutScore===1&&m++;f=g(l,m).pValue}let{sufficient:h}=w(n,n,i),_=a.lower>0||a.upper<0,v={mode:"controlled",baseline:t,delta:d,ci:a,pValue:f,significant:h&&_,sufficient:h,nWith:n,nWithout:n,costDelta:A(e,"withCost","withoutCost"),latencyDelta:A(e,"withLatency","withoutLatency"),confounded:!1};return h||(v.insufficientReason=`insufficient data: need >= ${i} paired tasks per cohort, have ${n}`),v}r(T,"buildControlledVerdict");export{T as a};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{a as k,b as A,c as I}from"../../chunk-47HDPK73.js";import{e as O}from"../../chunk-
|
|
1
|
+
import{a as k,b as A,c as I}from"../../chunk-47HDPK73.js";import{e as O}from"../../chunk-6N6NTMOR.js";import"../../chunk-TIOKY2IA.js";import"../../chunk-M4YF7YO7.js";import"../../chunk-RSKQRVPJ.js";import"../../chunk-35ST2BX7.js";import"../../chunk-6UTJGOBA.js";import"../../chunk-N7TGRFVZ.js";import"../../chunk-OSIP6FUD.js";import"../../chunk-ULRVMH7O.js";import"../../chunk-PRDYXO6Z.js";import"../../chunk-QFVGUZDO.js";import"../../chunk-T4P2NV3P.js";import"../../chunk-BUWWFHGO.js";import"../../chunk-MDPN46EY.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-CS2OQ3GC.js";import"../../chunk-DUP232AH.js";import"../../chunk-DO3KHK2X.js";import"../../chunk-UF6US7VU.js";import"../../chunk-MSTF7A7D.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-4HZWSFSY.js";import{k as u}from"../../chunk-KZ65LEUH.js";import"../../chunk-YQ56FQEO.js";import"../../chunk-QQ2UPNBC.js";import"../../chunk-TDD7IOV2.js";import"../../chunk-5DIOZDDZ.js";import"../../chunk-MODXEGCU.js";import"../../chunk-4VZLIGMS.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-HL755A5U.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-CJRYYV76.js";import"../../chunk-ZW5TAV3L.js";import"../../chunk-L2CFSMBC.js";import"../../chunk-Z72VHGV7.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-X2ILN3KP.js";import"../../chunk-HMMO27U4.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as b}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{f as N}from"../../chunk-H2ORWIEY.js";import{e as P,l as g}from"../../chunk-O6C2UGBY.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as te,context as re}from"@opentelemetry/api";import{NodeSDK as M}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as V}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as $,ConsoleSpanExporter as j,SimpleSpanProcessor as q}from"@opentelemetry/sdk-trace-node";import{Resource as F}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as G,ATTR_SERVICE_VERSION as U}from"@opentelemetry/semantic-conventions";function L(t){if(!t)return{};let r={};for(let n of t.split(",")){let o=n.indexOf("=");if(o===-1)continue;let i=n.slice(0,o).trim(),a=n.slice(o+1).trim();i&&(r[i]=a)}return r}e(L,"parseOtlpHeaders");var l=null;function y(){return process.env.TELEMETRY_ENABLED!=="false"}e(y,"isTelemetryEnabled");function h(t){if(!y()||l!==null)return"off";let r=new F({[G]:t.serviceName,[U]:t.serviceVersion??"0.1.0"}),n=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,o,i;if(n){let a=L(process.env.OTEL_EXPORTER_OTLP_HEADERS),d=new V({url:`${n}/v1/traces`,headers:a});i=new $(d),o="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")i=void 0,o="silent";else{let a=new j;i=new q(a),o="console"}return l=new M({resource:r,spanProcessors:i?[i]:[]}),l.start(),o}e(h,"initTelemetry");async function f(){l&&(await l.shutdown(),l=null)}e(f,"shutdownTelemetry");import{SpanKind as fe,SpanStatusCode as me}from"@opentelemetry/api";import{trace as Ee,context as Se}from"@opentelemetry/api";import*as p from"@sentry/node";var E=!1;function S(t){if(E)return!1;let r=process.env.SENTRY_DSN;return r?(p.init({dsn:r,serverName:t.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),E=!0,!0):!1}e(S,"initSentry");async function m(t){E&&await p.flush(t)}e(m,"flushSentry");import{existsSync as B,mkdirSync as H,readFileSync as w,unlinkSync as D,writeFileSync as C}from"node:fs";import{dirname as K,join as v}from"node:path";import{execFileSync as Y}from"node:child_process";var c=v(g(),"evals-agent.lock");function J(t){try{return process.kill(t,0),!0}catch(r){return r.code==="EPERM"}}e(J,"pidAlive");function z(t){if(!J(t))return!1;try{let r=Y("ps",["-p",String(t),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(r)}catch{return!0}}e(z,"isLiveAgent");function Q(){try{H(K(c),{recursive:!0})}catch{}for(let t=0;t<2;t++)try{return C(c,String(process.pid),{flag:"wx"}),{ok:!0}}catch(r){if(r.code!=="EEXIST")throw r;let n=0;try{n=Number(w(c,"utf-8").trim())}catch{}if(n&&n!==process.pid&&z(n))return{ok:!1,holder:n};try{D(c)}catch{}}try{C(c,String(process.pid))}catch{}return{ok:!0}}e(Q,"acquireAgentLock");function T(){try{B(c)&&Number(w(c,"utf-8").trim())===process.pid&&D(c)}catch{}}e(T,"releaseAgentLock");var x=class t extends b{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(t),this.requireAuth();let r=Q();if(!r.ok){this.log(`Another rulemetric evals agent is already running (pid ${r.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",S({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let o=h({serviceName:"rulemetric-agent"});o==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):o==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!P()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
|
|
2
2
|
`);let a=new AbortController,d=new k({providers:[A()],statePath:v(g(),"transcript-sync-state.json"),importTranscript:e(s=>N("/api/sessions/import",s),"importTranscript"),log:e(s=>this.log(`[transcript-sync] ${s}`),"log")});I(d,{signal:a.signal,log:e(s=>this.log(`[transcript-sync] ${s}`),"log")});let R=e(async s=>{this.log(`
|
|
3
3
|
[${s}] shutting down agent...`),a.abort(),await Promise.allSettled([m(2e3),f(),u()]),T(),process.exit(0)},"shutdown");process.on("SIGINT",()=>{R("SIGINT")}),process.on("SIGTERM",()=>{R("SIGTERM")});let _=!1;try{await O({signal:a.signal,log:e(s=>this.log(s),"log"),version:this.config.version,readInstalledVersion:e(()=>{try{return JSON.parse(w(v(this.config.root,"package.json"),"utf-8")).version??null}catch{return null}},"readInstalledVersion"),onVersionDrift:e(()=>{_=!0},"onVersionDrift")}),_&&(this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code."),await Promise.allSettled([m(2e3),f(),u()]),T(),process.exit(1))}finally{T(),await u()}}};export{x as default};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{b as h,c as f}from"../../chunk-
|
|
1
|
+
import{b as h,c as f}from"../../chunk-4O7MJUPH.js";import"../../chunk-X2ILN3KP.js";import{a as m}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as c}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as u}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as v}from"@oclif/core";var d=class b extends m{static{u(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={iteration:v.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:o,flags:l}=await this.parse(b);this.requireAuth();let g=await c(`/api/eval-targets/${o.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${l.iteration}`);let a=await h(o.targetId,l.iteration);a.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
|
|
2
2
|
Aggregated ${a.totalRuns} runs
|
|
3
3
|
`);for(let[e,n]of Object.entries(a.runSummary)){this.log(`${e}:`);for(let[r,i]of Object.entries(n)){let s=i;this.log(` ${r}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`)}}let t=a.verdict;if(t)if(this.log(`
|
|
4
4
|
Verdict (${t.mode}, baseline=${t.baseline}):`),!t.sufficient)this.log(` ${t.insufficientReason??"insufficient data"} \u2014 no effect claimed`);else{let e=t.delta>=0?"+":"";this.log(` pass-rate delta: ${e}${(t.delta*100).toFixed(0)}% (n=${t.nWith} paired)`),this.log(` 95% CI: ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%`),t.pValue!=null&&this.log(` McNemar p: ${t.pValue.toFixed(4)}`),this.log(` significant: ${t.significant?"YES":"no"}`)}if(a.notes.length>0){this.log(`
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{b as P,c as C}from"../../chunk-
|
|
1
|
+
import{b as P,c as C}from"../../chunk-WYF5BIQW.js";import{a as V}from"../../chunk-4VZLIGMS.js";import{a as N}from"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-X2ILN3KP.js";import{c as F}from"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as A}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as w,f as m,g as S}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as k}from"../../chunk-ZLLER3HP.js";import{createInterface as _}from"node:readline/promises";import{Args as q,Flags as g}from"@oclif/core";var $=class O extends A{static{k(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:q.string({description:"Eval target ID",required:!0})};static flags={auto:g.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":g.integer({description:"Maximum optimization iterations",default:5}),threshold:g.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:g.string({description:"Model for eval execution"}),"analyzer-model":g.string({description:"Model for analysis/suggestions"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:n,flags:e}=await this.parse(O);this.requireAuth();let t=await w(`/api/eval-targets/${n.targetId}`),i=await w(`/api/eval-targets/${n.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let s=[...i].sort(()=>Math.random()-.5),c=Math.ceil(s.length*.6),u=s.slice(0,c),d=s.slice(c);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${u.length} train, ${d.length} test`);let h=parseFloat(e.threshold);this.log(`Config: max ${e["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),e.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let T=t.version,f=0,x=0;for(let l=1;l<=e["max-iterations"];l++){this.log(`
|
|
2
2
|
=== Iteration ${l} ===`),this.log("Running train evals...");let b=await this.runEvalsOnTarget(t,u,e),p=this.computePassRate(b);if(this.log(`Train pass rate: ${(p*100).toFixed(1)}%`),p>=h){this.log(`Train score ${(p*100).toFixed(1)}% >= threshold ${(h*100).toFixed(0)}%. Stopping.`);let o=await this.runEvalsOnTarget(t,d,e),z=this.computePassRate(o);this.log(`Final test score: ${(z*100).toFixed(1)}%`),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,testScore:z,analysis:{reason:"threshold_reached"},status:"completed"});break}this.log("Analyzing results...");let M=b.map(o=>({evalName:o.evalName,prompt:o.prompt,expectations:o.expectations,grading:o.grading})),a=await V({content:t.content,evalResults:M,model:e["analyzer-model"]??e.model,timeout:e.timeout});this.log(`
|
|
3
3
|
Assessment: ${a.overallAssessment}`),this.log(`Suggestions: ${a.suggestions.length}`);for(let o of a.suggestions)this.log(` - [${o.section}] ${o.reasoning}`);if(a.proposedContent===t.content){this.log("No changes proposed. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,analysis:{reason:"no_changes"},status:"completed"});break}if(!e.auto&&!await this.confirm(`
|
|
4
4
|
Apply the proposed changes as a new version? [y/N] `)){this.log("Changes not applied. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,analysis:{reason:"user_declined",suggestions:a.suggestions},status:"completed"});break}let r=await S(`/api/eval-targets/${t.id}`,{content:a.proposedContent});this.log(`Created v${r.version}`),this.log(`Running test evals on v${t.version} (baseline) and v${r.version}...`);let E=await this.runEvalsOnTarget(t,d,e),R=await this.runEvalsOnTarget(r,d,e),v=this.computePassRate(R),I=this.computePassRate(E);this.log(`Test pass rate: v${t.version} ${(I*100).toFixed(1)}% \u2192 v${r.version} ${(v*100).toFixed(1)}%`);let y=P(E,R);if(y&&this.log(C(y,t.version,r.version)),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,outputVersion:r.version,trainScore:p,testScore:v,analysis:{suggestions:a.suggestions,overallAssessment:a.overallAssessment,priorTestScore:I,verdict:y},status:"completed"}),v>f)f=v,T=r.version,x=0;else if(x++,x>=2){this.log("No improvement for 2 consecutive iterations. Stopping.");break}t=r}this.log(`
|
package/dist/lib/agent-loop.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c,d,e}from"../chunk-
|
|
1
|
+
import{a,b,c,d,e}from"../chunk-6N6NTMOR.js";import"../chunk-TIOKY2IA.js";import"../chunk-M4YF7YO7.js";import"../chunk-RSKQRVPJ.js";import"../chunk-35ST2BX7.js";import"../chunk-6UTJGOBA.js";import"../chunk-N7TGRFVZ.js";import"../chunk-OSIP6FUD.js";import"../chunk-ULRVMH7O.js";import"../chunk-PRDYXO6Z.js";import"../chunk-QFVGUZDO.js";import"../chunk-T4P2NV3P.js";import"../chunk-BUWWFHGO.js";import"../chunk-MDPN46EY.js";import"../chunk-5RBLIVBF.js";import"../chunk-CS2OQ3GC.js";import"../chunk-DUP232AH.js";import"../chunk-DO3KHK2X.js";import"../chunk-UF6US7VU.js";import"../chunk-MSTF7A7D.js";import"../chunk-ZL4MZMUK.js";import"../chunk-4HZWSFSY.js";import"../chunk-KZ65LEUH.js";import"../chunk-YQ56FQEO.js";import"../chunk-QQ2UPNBC.js";import"../chunk-TDD7IOV2.js";import"../chunk-5DIOZDDZ.js";import"../chunk-MODXEGCU.js";import"../chunk-4VZLIGMS.js";import"../chunk-UUMW6X2W.js";import"../chunk-HL755A5U.js";import"../chunk-6UEFCMR2.js";import"../chunk-CJRYYV76.js";import"../chunk-ZW5TAV3L.js";import"../chunk-L2CFSMBC.js";import"../chunk-Z72VHGV7.js";import"../chunk-KAWNGQS2.js";import"../chunk-CDY7TQFZ.js";import"../chunk-K25S2MGU.js";import"../chunk-X2ILN3KP.js";import"../chunk-HMMO27U4.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c}from"../chunk-
|
|
1
|
+
import{a,b,c}from"../chunk-4O7MJUPH.js";import"../chunk-X2ILN3KP.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as aggregateBenchmark,c as postBenchmark,a as summarizeRuns};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-CS2OQ3GC.js";import"../../chunk-KZ65LEUH.js";import"../../chunk-KAWNGQS2.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-K25S2MGU.js";import"../../chunk-X2ILN3KP.js";import"../../chunk-HMMO27U4.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-ZLLER3HP.js";export{a as processMemoryAb};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c}from"../chunk-
|
|
1
|
+
import{a,b,c}from"../chunk-WYF5BIQW.js";import"../chunk-X2ILN3KP.js";import"../chunk-ZLLER3HP.js";export{b as buildVersionVerdict,c as formatVersionVerdict,a as pairVersionResults};
|
package/oclif.manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulemetric/cli",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.30",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -74,9 +74,9 @@
|
|
|
74
74
|
"postgres": "^3.4.0",
|
|
75
75
|
"react": "^19.0.0",
|
|
76
76
|
"zod": "^3.24.0",
|
|
77
|
-
"@rulemetric/hooks": "0.7.
|
|
78
|
-
"@rulemetric/
|
|
79
|
-
"@rulemetric/
|
|
77
|
+
"@rulemetric/hooks": "0.7.30",
|
|
78
|
+
"@rulemetric/skills-registry": "0.7.30",
|
|
79
|
+
"@rulemetric/proxy": "0.7.30"
|
|
80
80
|
},
|
|
81
81
|
"//deps": "Worker (`evals agent`) runtime deps: the @opentelemetry/* set, @sentry/node, and postgres are STATIC top-level imports loaded at worker startup (via bundled @rulemetric/{telemetry,db}) — they MUST be runtime deps or a clean `npm i -g` crashes with ModuleLoadError. `better-sqlite3` is optionalDependencies (below): it's a native module, lazy `require()`'d inside a try/catch (packages/session providers cursor/opencode) so a missing binary degrades gracefully — making it a hard dep would break `npm i -g` on platforms without prebuilt binaries. `resend` stays a devDep: lazy `await import('resend')` guarded by RESEND_API_KEY, which end-user workers never set (provider falls back to noop). Classification verified via import-graph scan of dist/commands/evals/agent.js + per-importer static/lazy check.",
|
|
82
82
|
"optionalDependencies": {
|
|
@@ -97,11 +97,11 @@
|
|
|
97
97
|
"@rulemetric/api": "0.0.1",
|
|
98
98
|
"@rulemetric/core": "0.0.1",
|
|
99
99
|
"@rulemetric/db": "0.0.1",
|
|
100
|
-
"@rulemetric/effectiveness": "0.0.1",
|
|
101
100
|
"@rulemetric/email": "0.0.1",
|
|
101
|
+
"@rulemetric/effectiveness": "0.0.1",
|
|
102
102
|
"@rulemetric/session": "0.2.1",
|
|
103
|
-
"@rulemetric/
|
|
104
|
-
"@rulemetric/
|
|
103
|
+
"@rulemetric/telemetry": "0.0.1",
|
|
104
|
+
"@rulemetric/typescript-config": "0.0.0"
|
|
105
105
|
},
|
|
106
106
|
"scripts": {
|
|
107
107
|
"build": "node scripts/build.mjs",
|