@rulemetric/cli 0.14.2 → 0.15.0

package/dist/{chunk-6AAAHV2B.js → chunk-3H44B6PB.js}

@@ -1 +1 @@
-import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-DVU5PHPM.js";import{d as J}from"./chunk-SG4UDDHB.js";import{b as B,c as R,d as N,e as z}from"./chunk-GPBN4ROE.js";import{e as f,g as T}from"./chunk-GIXHG5FK.js";import{a as p}from"./chunk-ZLLER3HP.js";function W(r,a,s){return s?r.reviewerModel===s||r.reviewerModel===`${a}:${s}`:r.reviewerEngine===a}p(W,"reviewMatchesReviewer");var C=100;function Z(r){let a=new Set;return r.forEach((s,d)=>{s.method!=="deterministic"&&a.add(d)}),a}p(Z,"llmExpectationIndexes");var w=class extends Error{constructor(s,d){super(d);this.reason=s}static{p(this,"JudgeReviewSkip")}};async function oe(r){let a=r.log??(()=>{}),s=r.warn??a,d=r.targetId,K=r.parallel??3,U=r.timeoutMs??12e4,v=await f(`/api/eval-targets/${d}`),u=(await f(`/api/eval-targets/${d}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new w("no-labels",`No human labels on "${v.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:r.engineFlag??"auto",modelFlag:r.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new w("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(r.skipReviewed){let e=await f(`/api/eval-targets/${d}/judge-reviews`),t=new Set(e.filter(i=>W(i,n.engine,r.modelFlag)).map(i=>`${i.evalRunId}:${i.expectationIndex}`));if(b=u.filter(i=>!t.has(`${i.evalRunId}:${i.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await f(`/api/eval-targets/${d}/runs?limit=${C}&offset=${e}`);for(let i of t)y.set(i.id,i);if(t.length<C)break}let q=await f(`/api/eval-targets/${d}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let x=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let i=Z(t),g=P.get(e)??new Set;[...i].every(h=>g.has(h))||(o.delete(e),x+=1)}if(x>0&&a(`${x} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:x};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&B(e.error)){if(n.model===R)throw new w("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);s(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new w("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${v.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let $=[],I=0,k=new Set,m=r.limit!=null?[...o.entries()].slice(0,Math.max(1,r.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=p(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,i]=m[e],g=y.get(t),h=g?.grading?.expectations,O=g?.outputs?.transcript,F=g?X.get(g.evalId):void 0;if(!g||!h?.length||!O||!F){I+=i.length,s(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:F,output:O,expectations:h.map(l=>l.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let l of i){let L=h[l.expectationIndex],A=M.expectations[l.expectationIndex];if(!L||!A||L.text!==l.expectationText){I+=1;continue}if(M.judge_model==null&&A.method==="llm"){I+=1;continue}$.push({runId:t,expectationIndex:l.expectationIndex,text:l.expectationText,humanPassed:D(l.verdict,l.judgePassed),judgeAPassed:l.judgePassed,judgeBPassed:A.passed})}a(`  ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j($,I),_=[...k][0]??n.model??`${n.engine} (CLI default)`,E=0;if($.length>0)try{E=(await T(`/api/eval-targets/${d}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:$.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${E} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){s(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:v.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:E,alreadyReviewed:c,deferredPartialRuns:x}}p(oe,"runJudgeReviewPass");export{W as a,Z as b,w as c,oe as d};
+import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-FG343JM3.js";import{d as J}from"./chunk-JNLDISBA.js";import{b as B,c as R,d as N,e as z}from"./chunk-GPBN4ROE.js";import{e as f,g as T}from"./chunk-GIXHG5FK.js";import{a as p}from"./chunk-ZLLER3HP.js";function W(r,a,s){return s?r.reviewerModel===s||r.reviewerModel===`${a}:${s}`:r.reviewerEngine===a}p(W,"reviewMatchesReviewer");var C=100;function Z(r){let a=new Set;return r.forEach((s,d)=>{s.method!=="deterministic"&&a.add(d)}),a}p(Z,"llmExpectationIndexes");var w=class extends Error{constructor(s,d){super(d);this.reason=s}static{p(this,"JudgeReviewSkip")}};async function oe(r){let a=r.log??(()=>{}),s=r.warn??a,d=r.targetId,K=r.parallel??3,U=r.timeoutMs??12e4,v=await f(`/api/eval-targets/${d}`),u=(await f(`/api/eval-targets/${d}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new w("no-labels",`No human labels on "${v.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:r.engineFlag??"auto",modelFlag:r.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new w("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(r.skipReviewed){let e=await f(`/api/eval-targets/${d}/judge-reviews`),t=new Set(e.filter(i=>W(i,n.engine,r.modelFlag)).map(i=>`${i.evalRunId}:${i.expectationIndex}`));if(b=u.filter(i=>!t.has(`${i.evalRunId}:${i.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await f(`/api/eval-targets/${d}/runs?limit=${C}&offset=${e}`);for(let i of t)y.set(i.id,i);if(t.length<C)break}let q=await f(`/api/eval-targets/${d}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let x=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let i=Z(t),g=P.get(e)??new Set;[...i].every(h=>g.has(h))||(o.delete(e),x+=1)}if(x>0&&a(`${x} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:x};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&B(e.error)){if(n.model===R)throw new w("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);s(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new w("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${v.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let $=[],I=0,k=new Set,m=r.limit!=null?[...o.entries()].slice(0,Math.max(1,r.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=p(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,i]=m[e],g=y.get(t),h=g?.grading?.expectations,O=g?.outputs?.transcript,F=g?X.get(g.evalId):void 0;if(!g||!h?.length||!O||!F){I+=i.length,s(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:F,output:O,expectations:h.map(l=>l.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let l of i){let L=h[l.expectationIndex],A=M.expectations[l.expectationIndex];if(!L||!A||L.text!==l.expectationText){I+=1;continue}if(M.judge_model==null&&A.method==="llm"){I+=1;continue}$.push({runId:t,expectationIndex:l.expectationIndex,text:l.expectationText,humanPassed:D(l.verdict,l.judgePassed),judgeAPassed:l.judgePassed,judgeBPassed:A.passed})}a(`  ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j($,I),_=[...k][0]??n.model??`${n.engine} (CLI default)`,E=0;if($.length>0)try{E=(await T(`/api/eval-targets/${d}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:$.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${E} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){s(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:v.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:E,alreadyReviewed:c,deferredPartialRuns:x}}p(oe,"runJudgeReviewPass");export{W as a,Z as b,w as c,oe as d};

package/dist/chunk-4DA6L7KL.js

@@ -0,0 +1 @@
+import{a as Y,b as F,c as T,d as Z,e as ee}from"./chunk-JU5ZNXPK.js";import{a as E}from"./chunk-5RBLIVBF.js";import{a as te}from"./chunk-KR36ZDGN.js";import{b as ne}from"./chunk-HXOQUYG3.js";import{a as z,c as X}from"./chunk-FADINV4H.js";import{f as N}from"./chunk-SWSOEYC7.js";import{a as K}from"./chunk-FG343JM3.js";import{a as B,f as Q}from"./chunk-4GOXEEXZ.js";import{e as R,f as I,h as $}from"./chunk-GIXHG5FK.js";import{a as w}from"./chunk-ZLLER3HP.js";import{execFileSync as ce}from"node:child_process";import{tmpdir as oe}from"node:os";import{z as e}from"zod";var C=e.string().regex(/^[a-f0-9]{64}$/),c=e.string().min(1).max(2e4),re=e.string().min(1).max(240).refine(r=>!r.startsWith("/")&&!r.includes("\\")&&r.split("/").every(p=>p!==".."&&p!=="."&&p!==""&&!p.startsWith(".")),"fixture paths must be relative, without traversal or hidden configuration"),P=e.object({path:re,content:e.string().max(1e5)}).strict(),se=e.object({name:c,path:re,kind:e.enum(["text_equals","json_equals","file_absent"]),expected:e.string().max(1e5),primary:e.boolean()}).strict(),le=e.object({evalId:e.string().uuid(),taskId:c,family:c,lineage:c,split:e.enum(["exploration","validation"]),incidentSessionIds:e.array(e.string().uuid()).min(1),opportunity:e.enum(["failure","success","costly_success","should_not_trigger"]),prompt:c,fixtureVersion:c,verifierVersion:c,files:e.array(P).max(100),checks:e.array(se).min(1).max(30),oracleFiles:e.array(P).max(100)}).strict(),ue=e.object({version:e.literal(1),revision:e.number().int().positive(),question:c,hypotheses:e.array(c).min(2),interventionType:e.enum(["instruction","retrieval","tool","executable_check","workflow","removal","none"]),baseline:e.object({content:e.string().max(16e3),files:e.array(P).max(30)}).strict(),treatment:e.object({content:e.string().max(16e3),files:e.array(P).max(30)}).strict(),scope:e.object({projectPath:c,harnessVersion:c,deliveryMode:e.literal("explicit-system-and-fixture-file"),engine:e.enum(["claude","codex"]),model:c,provider:e.enum(["anthropic","openai"]),repositoryRevision:e.string().regex(/^[a-f0-9]{40}$/),ambientContext:e.literal("frozen-fixture"),permissions:e.literal("isolated-fixture-write"),tools:e.array(c).min(1),expiresAt:e.string().datetime()}).strict(),sampling:e.object({rationale:c,exclusions:e.array(c),eligibleOpportunities:e.number().int().positive(),sampledOpportunities:e.number().int().positive()}).strict(),analysis:e.object({role:e.enum(["diagnostic","exploration","frozen_validation"]),primaryOutcome:c,safeguards:e.array(c).min(1),eligibility:c,assignmentUnit:e.literal("task"),weighting:e.literal("equal-task"),method:e.literal("paired-task-bootstrap"),stopping:e.literal("fixed-final"),primaryComparison:e.literal("treatment-minus-baseline"),repetitions:e.number().int().min(1).max(10),usefulEffect:e.number().positive().max(1),reliabilityMargin:e.number().min(0).max(1),maxCalls:e.number().int().min(2).max(200),maxSecondsPerCall:e.number().int().min(10).max(600),humanReviewMinutes:e.number().nonnegative().nullable(),reviewNote:c}).strict(),tasks:e.array(le).min(1).max(50),supersedes:e.string().uuid().nullable()}).strict().superRefine((r,p)=>{let s=w(i=>p.addIssue({code:"custom",message:i}),"fail");(new Set(r.tasks.map(i=>i.evalId)).size!==r.tasks.length||new Set(r.tasks.map(i=>i.taskId)).size!==r.tasks.length)&&s("task identity must be unique, not repetition"),r.scope.engine==="claude"!=(r.scope.provider==="anthropic")&&s("engine/provider mismatch"),r.scope.engine==="claude"&&[...r.scope.tools].sort().join(",")!=="Bash,Edit,Read,Write"&&s("Claude fixture execution supports exactly Read, Write, Edit and Bash");for(let i of[r.baseline,r.treatment])Array.from(i.content).reduce((n,g)=>{let d=g.codePointAt(0);return n+(d<128?1:d<2048?2:d<65536?3:4)},0)>16e3&&s("Instruction delivery exceeds the UTF-8 byte budget");let f=new Map;for(let i of r.tasks){for(let n of[i.lineage,i.family])f.has(n)&&f.get(n)!==i.split&&s("family or incident lineage leaks across splits"),f.set(n,i.split);[...i.files,...r.baseline.files,...r.treatment.files].some(n=>["CLAUDE.md","CLAUDE.local.md","AGENTS.md"].includes(n.path.split("/").at(-1)))&&s("Fixture files cannot introduce ambient instruction files"),new Set(i.checks.map(n=>n.name)).size!==i.checks.length&&s("Check names must be unique"),i.checks.some(n=>n.primary)||s("each task needs an independent primary artifact check"),new Set(i.files.map(n=>n.path)).size!==i.files.length&&s("duplicate fixture file path");for(let n of[r.baseline,r.treatment])new Set(n.files.map(g=>g.path)).size!==n.files.length&&s("duplicate intervention file path"),n.files.some(g=>i.files.some(d=>d.path===g.path))&&s("intervention must not overwrite common fixture files")}let l=r.tasks.filter(i=>i.split===(r.analysis.role==="frozen_validation"?"validation":"exploration"));l.length||s("no tasks in the selected split"),l.length*2*r.analysis.repetitions>r.analysis.maxCalls&&s("planned calls exceed resource ceiling"),r.sampling.sampledOpportunities>r.sampling.eligibleOpportunities&&s("sampled opportunities exceed eligible opportunities")}),ie=e.object({version:e.literal(1),id:e.string().uuid(),protocolHash:C,protocol:ue}).strict(),de=e.object({studyId:e.string().uuid(),protocolHash:C,taskId:c,arm:e.enum(["baseline","treatment"]),repetition:e.number().int().positive(),eligible:e.boolean(),exclusion:e.string().nullable(),primaryPassed:e.boolean(),safeguardsPassed:e.boolean(),checks:e.array(e.object({name:c,passed:e.boolean(),actual:e.string().nullable()})),deliveryHash:C,fixtureHash:C,harnessVersion:e.string().nullable(),engine:e.string().nullable(),model:e.string().nullable(),provider:e.string().nullable(),durationMs:e.number().nonnegative(),tokens:e.number().nonnegative().nullable(),costUsd:e.number().nonnegative().nullable()}).strict();async function _e(r,p){let{mirrorId:s,evalTargetId:f}=r,l=r.measurement?{...r.config,...X({...r.config,measurement:r.measurement})}:r.config,i=w(t=>p.logger.info(t),"log"),n=r.study?ie.parse(r.study):null;if(n&&n.protocol.scope.engine!=="claude")throw new E("Frozen study isolation is currently verified only for Claude");if(n&&Y(n.protocol)!==n.protocolHash)throw new E("Study snapshot hash mismatch");let g=await R(`/api/eval-targets/${f}`),d=await R(`/api/eval-targets/${f}/evals`);if(n&&(d=n.protocol.tasks.filter(t=>t.split===(n.protocol.analysis.role==="frozen_validation"?"validation":"exploration")).map(t=>({id:t.evalId,name:t.taskId,prompt:t.prompt,expectedOutput:"",expectations:[]}))),d.length===0&&l.generateIfEmpty){i("No eval cases on target \u2014 generating from content via local LLM...");let t=await ne({content:g.content,name:g.name,...r.measurement?{measurement:r.measurement}:{}});if(t===null)throw new Error("Case generation requires a local LLM backend (Claude or Codex CLI) \u2014 none available on this worker.");let a=[];for(let o of t)a.push(await I(`/api/eval-targets/${f}/evals`,o));d=a,i(`Generated ${a.length} eval case(s)`)}if(d.length===0)throw new E("No evals defined for target");let M=null;if(n&&(M=ce(n.protocol.scope.engine,["--version"],{encoding:"utf8",timeout:1e4}).trim(),M!==n.protocol.scope.harnessVersion))throw new E("Observed harness version differs from frozen protocol");if(n){if(l.engine!==n.protocol.scope.engine||l.model!==n.protocol.scope.model||l.runsPerEval!==n.protocol.analysis.repetitions||l.timeout!==n.protocol.analysis.maxSecondsPerCall*1e3||l.skipBaseline||new Date(n.protocol.scope.expiresAt)<=new Date)throw new E("Study execution configuration differs from frozen protocol or scope expired");for(let t of n.protocol.tasks.filter(a=>d.some(o=>o.id===a.evalId))){let a=N({baseDir:oe(),engine:n.protocol.scope.engine});try{F(a.workDir,[...t.files,...t.oracleFiles]);let o=T(a.workDir,t);if(!o.primaryPassed||!o.safeguardsPassed)throw new E(`Oracle failed for ${t.taskId}; no actor calls started`)}finally{a.cleanup()}}}let ae=l.runsPerEval??1,D=l.timeout??12e4,H=n?1:l.iteration??1,_=new Set;if(p.job.attempts>1)for(let a=0;;a+=100){let o=await R(`/api/eval-targets/${f}/runs?jobId=${s}&limit=100&offset=${a}`);for(let m of o)_.add(`${m.evalId}|${m.configuration}|${m.runNumber}|${m.iteration}`);if(o.length<100)break}let O=[{name:"with_target",content:n?.protocol.treatment.content??g.content}];l.skipBaseline||O.push({name:"without_target",...n?{content:n.protocol.baseline.content}:{}});let b=[];for(let t of d)for(let a of O)for(let o=1;o<=ae;o++)_.has(`${t.id}|${a.name}|${o}|${H}`)||b.push({eval_:t,configuration:a.name,targetContent:a.content,runNumber:o});n&&b.sort((t,a)=>t.eval_.id.localeCompare(a.eval_.id)||t.runNumber-a.runNumber||((d.findIndex(o=>o.id===t.eval_.id)+t.runNumber)%2?t.configuration.localeCompare(a.configuration):a.configuration.localeCompare(t.configuration))),_.size>0&&i(`Resuming attempt ${p.job.attempts}: ${_.size} run(s) already persisted, ${b.length} remaining`);let h=b.length,y=0,x=B(l.engine),k=Math.min(l.parallel??1,4);await $(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:h,completed:0,current:null,message:`Starting ${h} eval runs (concurrency: ${k})...`}});let S=[];async function A(t){let a=`${t.configuration} \xB7 ${t.eval_.name} \xB7 run#${t.runNumber}`;i(`[${y+1}/${h}] ${a}`);let o=N({...n?{baseDir:oe()}:{},targetType:n?"instruction":g.type,targetContent:t.targetContent,targetName:g.name,engine:x});o.leakedFiles.length&&i(`\u26A0 CONTAMINATED ${a}: ${o.leakedFiles.join(", ")} survived the strip`);try{let m=n?.protocol.tasks.find(W=>W.evalId===t.eval_.id),v=t.configuration==="with_target"?"treatment":"baseline";n&&m&&F(o.workDir,[...m.files,...n.protocol[v].files]);let q=n&&m?ee(o.workDir,m,n.protocol[v],x):null;n&&m&&await I(`/api/studies/${n.id}/calls`,{jobId:s,taskId:m.taskId,arm:v,repetition:t.runNumber});let u=await Q({prompt:t.eval_.prompt,targetContent:n?t.targetContent:o.systemPrompt,...n?{appendTarget:!0}:{},workDir:o.workDir,timeout:D,model:l.model,engine:x,...n?{writableFixture:!0}:{}}),G={};!l.skipGrading&&t.eval_.expectations.length>0&&!u.error&&(G=await K({...z({engine:x,model:l.model},{engine:l.graderEngine,model:l.graderModel}),prompt:t.eval_.prompt,output:u.output,expectations:t.eval_.expectations,timeout:D}));let L=n?Z(n.protocol,u):null,V=n&&m?{studyId:n.id,protocolHash:n.protocolHash,taskId:m.taskId,arm:v,repetition:t.runNumber,eligible:L===null,exclusion:L,...T(o.workDir,m),deliveryHash:q.deliveryHash,fixtureHash:q.fixtureHash,harnessVersion:M,engine:u.engine??null,model:u.model??null,provider:u.engine==="claude"?"anthropic":u.provider??null,durationMs:u.durationMs,tokens:u.totalTokens||null,costUsd:u.costUsd??null}:null;return{jobId:s,evalId:t.eval_.id,evalTargetId:g.id,evalTargetVersion:g.version,configuration:t.configuration,runNumber:t.runNumber,iteration:H,status:u.error?"failed":"completed",outputs:{...V?{study:V}:{},...u.usage?{usage:u.usage}:{},...u.error?{error:u.error}:{transcript:u.output,...u.model?{model:u.model}:{}},methodology:o.methodology,delivery:o.delivery,isolated:o.isWorktree,engine:u.engine??x,model:u.model??null,deliveryRegime:`${g.type}:${x}:${o.methodology}`,...r.measurement?{measurement:r.measurement}:{},...u.provider?{provider:u.provider}:{},...u.costUsd!=null?{costUsd:u.costUsd}:{},...o.leakedFiles.length?{leakedFiles:o.leakedFiles}:{}},timing:{duration_ms:u.durationMs,total_tokens:u.totalTokens},grading:G,worktree:o.isWorktree?!0:void 0}}finally{o.cleanup()}}if(w(A,"processItem"),k<=1)for(let t of b){let a=`${t.configuration} \xB7 ${t.eval_.name} \xB7 run#${t.runNumber}`;await $(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:h,completed:y,current:a,message:`Executing ${a}...`}});let o=await A(t);await I("/api/eval-runs",o),S.push(o),y++,await $(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:h,completed:y,current:a,message:`Completed ${a}`}})}else{i(`Running ${h} evals with concurrency=${k} (worktree isolation)`);for(let t=0;t<b.length;t+=k){let a=b.slice(t,t+k),o=`batch ${Math.floor(t/k)+1}/${Math.ceil(b.length/k)}`;await $(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:h,completed:y,current:o,message:`Executing ${o} (${a.length} parallel)...`}});let m=await Promise.all(a.map(A));for(let v of m)await I("/api/eval-runs",v),S.push(v),y++;await $(`/api/eval-jobs/${s}/progress`,{status:"running",progress:{total:h,completed:y,current:o,message:`Completed ${o}`}})}}let J=me(S),j;if(!l.skipGrading&&S.some(t=>t.grading&&Object.keys(t.grading).length>0))try{i(`Running meta-judge for job ${s}...`);let t=d.map(a=>{let m=S.filter(v=>v.evalId===a.id&&v.configuration==="with_target")[0];return{evalName:a.name,prompt:a.prompt,expectations:a.expectations,grading:m?.grading?m.grading:null,output:m?.outputs?.transcript?.slice(0,500)}});j=await te({...z({engine:x,model:l.model},{engine:l.graderEngine,model:l.graderModel}),targetName:g.name,evalResults:t,timeout:6e4}),i(`Meta-judge: quality=${j.quality_score}/5, issues=${j.issues.length}`)}catch(t){i(`Meta-judge failed (non-fatal): ${t instanceof Error?t.message:String(t)}`)}let U=j?{...J,meta_judgement:j}:J;return i(`Job ${s} completed: ${y}/${h} runs`),await $(`/api/eval-jobs/${s}/state`,{result:U}).catch(t=>{i(`result persist failed: ${t instanceof Error?t.message:String(t)}`)}),U}w(_e,"processEval");function me(r){let p=r.filter(i=>i.status==="completed"),s=r.filter(i=>i.status==="failed"),f=p.filter(i=>{let n=i.grading;return n&&Object.keys(n).length>0}),l=f.length>0?f.reduce((i,n)=>i+(n.grading?.summary?.pass_rate??0),0)/f.length:null;return{total:r.length,completed:p.length,failed:s.length,avgPassRate:l}}w(me,"buildSummary");export{_e as a};

package/dist/chunk-4GOXEEXZ.js

@@ -0,0 +1 @@
+import{d as P}from"./chunk-JP6ULOEX.js";import{d as k}from"./chunk-JNLDISBA.js";import{c as b,d as _}from"./chunk-GPBN4ROE.js";import{a as u}from"./chunk-ZLLER3HP.js";import{spawn as M}from"node:child_process";var E=["claude","codex","pi"];function N(e,t=process.env){if(e){if(E.includes(e))return e;throw new Error(`Eval engine "${String(e)}" is not known (expected one of: ${E.join(", ")})`)}let r=t.RULEMETRIC_EVAL_ENGINE?.trim().toLowerCase();if(!r)return"claude";if(E.includes(r))return r;throw new Error(`RULEMETRIC_EVAL_ENGINE="${t.RULEMETRIC_EVAL_ENGINE}" is not a known engine (expected one of: ${E.join(", ")})`)}u(N,"resolveEvalEngine");function S(e,t=process.env){return e?.trim()||t.RULEMETRIC_CODEX_MODEL?.trim()||_().codexConfiguredModel||b}u(S,"resolveCodexEvalModel");function h(e){try{let t=JSON.parse(e),r=t.result??t.content??e,a=t.usage?.total_tokens??(t.usage?.input_tokens??0)+(t.usage?.output_tokens??0)+(t.usage?.cache_creation_input_tokens??0)+(t.usage?.cache_read_input_tokens??0),n;if(t.modelUsage&&typeof t.modelUsage=="object"){let o=-1;for(let[l,d]of Object.entries(t.modelUsage)){let i=typeof d?.outputTokens=="number"?d.outputTokens:0;i>o&&(o=i,n=l)}}return{output:r,totalTokens:a,model:n,...t.usage&&typeof t.usage=="object"&&!Array.isArray(t.usage)?{usage:t.usage}:{}}}catch{return{output:e,totalTokens:0}}}u(h,"parseClaudeJsonOutput");function F(e){let{prompt:t,targetContent:r,systemPrompt:a,model:n}=e,o=["-p",t,"--output-format","json","--settings",JSON.stringify({disableAllHooks:!0,...e.writableFixture?{permissions:{blockReadsOutsideWorkingDirectories:!0},sandbox:{enabled:!0,failIfUnavailable:!0,autoAllowBashIfSandboxed:!0,allowUnsandboxedCommands:!1,excludedCommands:[],filesystem:{disabled:!1},network:{allowedDomains:[],strictAllowlist:!0}}}:{}})];e.writableFixture&&o.push("--safe-mode","--restricted","--strict-mcp-config","--tools","Read,Write,Edit,Bash","--permission-mode","acceptEdits","--allowedTools","Read,Write,Edit,Bash");let l=r??a;return l&&o.push(e.appendTarget?"--append-system-prompt":"--system-prompt",l),n&&o.push("--model",n),o}u(F,"buildClaudeArgs");function I(e,t,r,a){let n=t.trim()||r.trim()||a.trim()||"(no output on stderr or stdout)";return`Exit code ${e}: ${n.slice(0,1e3)}`}u(I,"describeExitFailure");async function U(e){let{prompt:t,targetContent:r,systemPrompt:a,workDir:n,timeout:o,model:l}=e,d=r??a,i=await k({prompt:t,systemPrompt:d,model:S(l),writableFixture:e.writableFixture,timeout:o,workDir:n});return{output:i.output,exitCode:i.exitCode,durationMs:i.durationMs,totalTokens:i.usage?.totalTokens??0,model:i.model,engine:"codex",delivery:d?"prompt_prefix":"none",error:i.error}}u(U,"executeCodexArm");async function j(e){let t=e.targetContent??e.systemPrompt,r=await P({prompt:e.prompt,...e.appendTarget?{appendSystemPrompt:t}:{systemPrompt:t},ignoreContextFiles:e.ignoreContextFiles,model:e.model,timeout:e.timeout,workDir:e.workDir});return{output:r.output,exitCode:r.exitCode,durationMs:r.durationMs,totalTokens:r.usage?.totalTokens??0,model:r.model,provider:r.provider,costUsd:r.costUsd,engine:"pi",delivery:t?"system_prompt":"none",error:r.error}}u(j,"executePiArm");async function J(e){let{prompt:t,targetContent:r,systemPrompt:a,workDir:n=process.cwd(),timeout:o=12e4,model:l}=e,d=N(e.engine);if(d==="codex")return U({...e,workDir:n,timeout:o});if(d==="pi")return j({...e,workDir:n,timeout:o});let i=F({prompt:t,targetContent:r,systemPrompt:a,model:l,appendTarget:e.appendTarget,writableFixture:e.writableFixture}),g=r??a?"system_prompt":"none",x=Date.now();return new Promise(f=>{let c=e.writableFixture?Object.fromEntries(Object.entries(process.env).filter(([s])=>["HOME","PATH","USER","LOGNAME","SHELL","TMPDIR","LANG","LC_ALL","TZ","SSL_CERT_FILE","NODE_EXTRA_CA_CERTS","ANTHROPIC_API_KEY","CLAUDE_CODE_OAUTH_TOKEN","CLAUDE_CONFIG_DIR"].includes(s))):{...process.env};delete c.CLAUDECODE,delete c.HTTPS_PROXY;let D=["/opt/homebrew/bin","/usr/local/bin"],C=(c.PATH??"").split(":").filter(Boolean),v=D.filter(s=>!C.includes(s));v.length&&(c.PATH=[...C,...v].join(":"));let p=M("claude",i,{cwd:n,env:c,stdio:["ignore","pipe","pipe"]}),m="",T="";p.stdout.on("data",s=>{m+=s.toString()}),p.stderr.on("data",s=>{T+=s.toString()});let w=setTimeout(()=>{p.kill("SIGTERM"),f({output:m,exitCode:null,durationMs:Date.now()-x,totalTokens:0,engine:"claude",delivery:g,error:`Timed out after ${o}ms`})},o);p.on("close",s=>{clearTimeout(w);let R=Date.now()-x,{output:y,totalTokens:A,model:O,usage:L}=h(m);f({output:y,exitCode:s,durationMs:R,totalTokens:A,usage:L,model:O,engine:"claude",delivery:g,error:s!==0?I(s,T,y,m):void 0})}),p.on("error",s=>{clearTimeout(w),f({output:"",exitCode:null,durationMs:Date.now()-x,totalTokens:0,engine:"claude",delivery:g,error:`Failed to spawn claude: ${s.message}`})})})}u(J,"executeEval");export{N as a,S as b,h as c,F as d,I as e,J as f};

package/dist/{chunk-VCUW2NAP.js → chunk-5Z6XXOJL.js}

@@ -1,4 +1,4 @@
-import{g as E}from"./chunk-7BCZDVIU.js";import{b,e as y,f as k,h as M}from"./chunk-GIXHG5FK.js";import{a as u}from"./chunk-ZLLER3HP.js";var v=50,I=24e3,P=3,R=3,C=40,N=4e3,x=new Set(["fact","decision","correction","convention"]),L=`You are distilling DURABLE, PROJECT-SPECIFIC knowledge from an AI coding session so a future agent working on the SAME project starts smarter.
+import{g as E}from"./chunk-XCH4HPEZ.js";import{b,e as y,f as k,h as M}from"./chunk-GIXHG5FK.js";import{a as u}from"./chunk-ZLLER3HP.js";var v=50,I=24e3,P=3,R=3,C=40,N=4e3,x=new Set(["fact","decision","correction","convention"]),L=`You are distilling DURABLE, PROJECT-SPECIFIC knowledge from an AI coding session so a future agent working on the SAME project starts smarter.
 
 Extract ONLY facts / decisions / corrections / conventions that will STILL be true and useful in FUTURE sessions \u2014 architecture facts, deliberate design decisions, corrections of a wrong approach, or project conventions.
 

package/dist/{chunk-Q36QSFSB.js → chunk-6QMIDH6X.js}

@@ -1,4 +1,4 @@
-import{f as p}from"./chunk-FBSZ6KCM.js";import{a as l}from"./chunk-ZLLER3HP.js";var k=[{name:"git-commit-helper",description:"Writes conventional commit messages and PR descriptions from a diff"},{name:"sql-query-optimizer",description:"Analyzes slow SQL queries and proposes indexes and rewrites"},{name:"i18n-string-extractor",description:"Finds hardcoded UI strings and extracts them into translation files"}],T=`You are an AI coding assistant with access to a set of optional skills. When a user request matches a skill's description, you invoke that skill before answering; when no skill applies, you answer directly without one. Descriptions are your ONLY information about each skill.
+import{f as p}from"./chunk-4GOXEEXZ.js";import{a as l}from"./chunk-ZLLER3HP.js";var k=[{name:"git-commit-helper",description:"Writes conventional commit messages and PR descriptions from a diff"},{name:"sql-query-optimizer",description:"Analyzes slow SQL queries and proposes indexes and rewrites"},{name:"i18n-string-extractor",description:"Finds hardcoded UI strings and extracts them into translation files"}],T=`You are an AI coding assistant with access to a set of optional skills. When a user request matches a skill's description, you invoke that skill before answering; when no skill applies, you answer directly without one. Descriptions are your ONLY information about each skill.
 
 {{CATALOG}}
 

package/dist/{chunk-WIKCYUZC.js → chunk-74KE4VOX.js}

@@ -1 +1 @@
-import{b as t,e as E}from"./chunk-3NWJ7CZ2.js";import{a as r}from"./chunk-ZLLER3HP.js";function s(o=10,_=t){for(let e=1;e<=o;e++)if(E(e,o,_).feasible)return e;return o}r(s,"minFeasibleLiveCases");var S=s(),A=3;export{s as a,S as b,A as c};
+import{b as t,e as E}from"./chunk-DYET4EYU.js";import{a as r}from"./chunk-ZLLER3HP.js";function s(o=10,_=t){for(let e=1;e<=o;e++)if(E(e,o,_).feasible)return e;return o}r(s,"minFeasibleLiveCases");var S=s(),A=3;export{s as a,S as b,A as c};

package/dist/{chunk-6TGFU6RU.js → chunk-7LFCZI64.js}

@@ -1,4 +1,4 @@
-import{h as E}from"./chunk-7BCZDVIU.js";import{a as r}from"./chunk-ZLLER3HP.js";var g="<<<FILE-CONTENT-BEGIN>>>",m="<<<FILE-CONTENT-END>>>",b="<RATIONALE>",R="</RATIONALE>",u="<PROPOSED_INSTRUCTION>",p="</PROPOSED_INSTRUCTION>",d=.5,f=3;function y(e){let t=e.immutableSections.length>0?e.immutableSections.map(s=>`  - ${s}`).join(`
+import{h as E}from"./chunk-XCH4HPEZ.js";import{a as r}from"./chunk-ZLLER3HP.js";var g="<<<FILE-CONTENT-BEGIN>>>",m="<<<FILE-CONTENT-END>>>",b="<RATIONALE>",R="</RATIONALE>",u="<PROPOSED_INSTRUCTION>",p="</PROPOSED_INSTRUCTION>",d=.5,f=3;function y(e){let t=e.immutableSections.length>0?e.immutableSections.map(s=>`  - ${s}`).join(`
 `):"  (none)",o=e.evidence.priorDecisions.length>0?["6. You have edited this file before. The attempts are listed below with","   how each was judged. Do not re-propose an edit that was MEASURED and","   rejected: it was replayed against this exact file and did not beat it,","   so measuring it again spends the same budget for the same answer. An","   attempt marked NOT MEASURED is different \u2014 the gate stopped it before","   it earned a verdict, so whether it would have helped is still unknown."]:[],n=e.currentContent.trim().length,i=`   It is currently ${n} characters. Your proposal must stay between ${Math.ceil(n*d)} and ${Math.floor(n*f)} characters \u2014 outside that it is discarded unmeasured as a truncation or runaway rewrite.`;return["You are improving an AI coding assistant instruction file so that it scores","better on a fixed set of evaluation cases.","",`The file is a ${e.targetType} named "${e.targetName}".`,i,"","RULES",`0. The current file is EXACTLY the text between ${g} and`,`   ${m} in the data message. Headings after the closing`,"   sentinel (underperforming cases, notes, prior attempts) are evidence",'   ABOUT the file \u2014 they are not in the file, and "removing" one of them',"   is not an edit. Do not include the sentinels in your output.","1. Return the COMPLETE new file, not a diff, not a patch, not a summary,","   and not an excerpt. Everything you omit will be DELETED from the file.","2. Preserve everything you are not deliberately changing, byte for byte.","3. Make a focused edit. You are one step in a measured loop: a small change","   that can be attributed beats a rewrite that cannot.","4. Do not edit these sections or anything nested under them \u2014 an edit that","   touches them is refused downstream and the whole proposal is wasted:",t,"5. Do not add meta-commentary about the evaluation, the loop, or this","   prompt to the file. The file is read by other agents as instructions.",...o,"","OUTPUT FORMAT \u2014 exactly this, with no text before or after:",b,"One or two sentences: what you changed and which evidence motivated it.",R,u,"The complete new file content.",p].join(`
 `)}r(y,"buildProposerSystemPrompt");var h=r(e=>`${e>=0?"+":"-"}${Math.abs(e*100).toFixed(1)}pp`,"signedPp");function O(e){let t=/^\d{4}-\d{2}-\d{2}/.exec(e.at??"")?.[0]??"undated",o;if(e.measured){let i=e.measured;o=`MEASURED against this exact file and ${e.decision==="promote"?"ACCEPTED":"REJECTED"}: ${h(i.delta)} (CI ${h(i.ciLower)} to ${h(i.ciUpper)}, ${i.nPairs} paired cases)`}else o=`NOT MEASURED \u2014 the gate refused it, so its merit is still unknown: ${e.refusals.length>0?e.refusals.join("; "):"no reason recorded"}`;let n=[`- ${t} \u2014 ${o}`];return e.decision==="promote"&&n.push(e.applied?"  The file WAS rewritten by this decision.":"  Dry run: the file was NOT rewritten, so this edit is still available."),e.touched.length>0&&n.push(`  Sections touched: ${e.touched.join(", ")}`),e.rationale.trim()!==""&&n.push(`  You said at the time: ${e.rationale.trim()}`),n.join(`
 `)}r(O,"describePriorDecision");function P(e){let t=e.evidence,o=["## CURRENT FILE CONTENT","","The file is EXACTLY the text between the two sentinel lines below \u2014","nothing before the first sentinel or after the second is part of it.","Everything after the closing sentinel is evidence ABOUT the file.","",g,e.currentContent,m,""];if(t.weakest.length>0){o.push("## CASES WHERE THIS FILE CURRENTLY UNDERPERFORMS","");for(let n of t.weakest)o.push(`- ${n.name}: passes ${(n.withPassRate*100).toFixed(0)}% with the file vs ${(n.withoutPassRate*100).toFixed(0)}% without it`);o.push("")}return t.nonDiscriminating.length>0&&o.push("## CASES THAT CARRY NO SIGNAL EITHER WAY","","These score the same with and without the file. Changing the file to chase","them is unlikely to move anything.","",...t.nonDiscriminating.map(n=>`- ${n}`),""),t.priorDecisions.length>0&&o.push("## WHAT YOU ALREADY PROPOSED FOR THIS FILE, AND HOW IT WAS JUDGED","","Newest first. Every attempt listed here was made against the file above,","byte for byte. Attempts against a different version of the file are not","shown, because a verdict earned by different text says nothing about this","one.","",...t.priorDecisions.map(O),""),t.notes.length>0&&o.push("## NOTES","",...t.notes.map(n=>`- ${n}`),""),o.join(`

package/dist/{chunk-7H4HJ2RV.js → chunk-AFEJYDUK.js}

@@ -1 +1 @@
-import{b as l}from"./chunk-VBXL7XO7.js";import{a as r}from"./chunk-ZLLER3HP.js";var u=r(n=>{let t=n.grading.summary?.pass_rate;return typeof t=="number"?t:null},"passRateOf"),a=r(n=>(u(n)??0)>=1,"fullPass");function f(n,t){let i=new Map(n.filter(e=>u(e)!=null).map(e=>[e.evalName,e])),o=[];for(let e of t){if(u(e)==null)continue;let s=i.get(e.evalName);s&&o.push({withScore:a(e)?1:0,withoutScore:a(s)?1:0})}return o}r(f,"pairVersionResults");function d(n,t){let i=f(n,t);return i.length===0?null:l(i,{baseline:"prior-version"})}r(d,"buildVersionVerdict");function m(n,t,i){let o=`v${i} vs v${t}`;if(!n.sufficient)return`${o}: ${n.insufficientReason??"insufficient data"} \u2014 no effect claimed`;let e=n.delta>=0?"+":"",s=`95% CI ${(n.ci.lower*100).toFixed(0)}\u2013${(n.ci.upper*100).toFixed(0)}%`,c=n.significant?"significant":"not significant";return`${o}: ${e}${(n.delta*100).toFixed(0)}% pass rate (${s}, n=${n.nWith} paired, ${c})`}r(m,"formatVersionVerdict");export{f as a,d as b,m as c};
+import{b as l}from"./chunk-HT6R6KR3.js";import{a as r}from"./chunk-ZLLER3HP.js";var u=r(n=>{let t=n.grading.summary?.pass_rate;return typeof t=="number"?t:null},"passRateOf"),a=r(n=>(u(n)??0)>=1,"fullPass");function f(n,t){let i=new Map(n.filter(e=>u(e)!=null).map(e=>[e.evalName,e])),o=[];for(let e of t){if(u(e)==null)continue;let s=i.get(e.evalName);s&&o.push({withScore:a(e)?1:0,withoutScore:a(s)?1:0})}return o}r(f,"pairVersionResults");function d(n,t){let i=f(n,t);return i.length===0?null:l(i,{baseline:"prior-version"})}r(d,"buildVersionVerdict");function m(n,t,i){let o=`v${i} vs v${t}`;if(!n.sufficient)return`${o}: ${n.insufficientReason??"insufficient data"} \u2014 no effect claimed`;let e=n.delta>=0?"+":"",s=`95% CI ${(n.ci.lower*100).toFixed(0)}\u2013${(n.ci.upper*100).toFixed(0)}%`,c=n.significant?"significant":"not significant";return`${o}: ${e}${(n.delta*100).toFixed(0)}% pass rate (${s}, n=${n.nWith} paired, ${c})`}r(m,"formatVersionVerdict");export{f as a,d as b,m as c};

package/dist/{chunk-QENSCK77.js → chunk-CIKQXFQ2.js}

@@ -1 +1 @@
-import{c as X}from"./chunk-HOQPVOMF.js";import{b as D}from"./chunk-GIRPEMTW.js";import{a as x}from"./chunk-MODXEGCU.js";import{e as J,g as B}from"./chunk-7BCZDVIU.js";import{e as w,f as R}from"./chunk-GIXHG5FK.js";import{c as H,d as U}from"./chunk-X6CF5KTW.js";import{a as u}from"./chunk-ZLLER3HP.js";import{readFileSync as q,realpathSync as V}from"node:fs";import y from"node:path";import A from"node:os";function Q(s){let n=y.resolve(s);if(n===y.resolve(A.homedir()))return!1;let a=y.parse(n).root;if(n===a)return!1;try{let o=V(A.tmpdir());if(V(n)===o)return!1}catch{}return!0}u(Q,"isScorableProjectPath");var Z=5,ee=Number(process.env.RULEMETRIC_MAX_SUGGESTION_PROJECTS??10),te=48,ne=.5,O=400,oe=3,re=.6,se=1;function ae(s){let n=s.bloated??[],a=new Set([...s.harmful.map(o=>o.id),...s.candidates.map(o=>o.id)]);return n.filter(o=>!a.has(o.id)).slice(0,se).map(o=>({instructionId:o.id,estimatedTokens:o.estimatedTokens,topProjectId:o.topProjectId??null,score:.55,reason:`Costs ~${o.estimatedTokens} tokens every session and measures neutral across ${o.exposures} exposed session(s) \u2014 enough data to detect an effect, and none found. Consider shortening "${o.name}" rather than removing it.`+(o.containsSafetyLanguage?" CAUTION: it contains guardrail language (destructive-command or credential rules) \u2014 preserve those lines when shortening.":"")}))}u(ae,"buildTrimProposals");async function ie(s,n,a){let o=[],i=0;for(let d of s)try{await a("/api/instruction-experiments",{instructionId:d.instructionId,projectPath:n,source:"groom"}),i++}catch(g){let m=g instanceof Error?g.message:String(g);o.push({...d,reason:`${d.reason} (not measurable by withholding: ${m})`})}return{nominated:i,unmeasurable:o}}u(ie,"nominateRemovals");function ce(s){let n=[];for(let o of s.harmful){let i=o.verdict.delta;n.push({instructionId:o.id,score:Math.min(.95,.75+Math.abs(i)),topProjectId:o.topProjectId??null,reason:`Measured harmful: sessions using "${o.name}" scored ${(i*100).toFixed(1)}pp worse across ${o.exposures} exposed session(s). Observational \u2014 review before archiving.`})}let a=new Set(s.harmful.map(o=>o.id));for(let o of s.candidates)a.has(o.id)||n.push({instructionId:o.id,score:re,topProjectId:o.topProjectId??null,reason:`Unused: "${o.name}" is ${o.reason} (${o.totalExposures} lifetime exposure(s)) but still costs context budget every session.`});return n.slice(0,oe)}u(ce,"buildRemovalProposals");function le(){let s=y.join(A.homedir(),".config","rulemetric","skills-registry.json");try{let n=q(s,"utf-8"),a=JSON.parse(n),o=new Date(a.generatedAt).getTime();return Number.isNaN(o)?"unknown":Date.now()-o>te*60*60*1e3?"stale":"fresh"}catch{return"unknown"}}u(le,"observeSkillsRegistry");var Re=u(async(s,n)=>{let a=J(),o=a!=="none";o?n.logger.info(`Scoring backend: ${a}`):n.logger.warn("No LLM backend available (claude CLI missing and ANTHROPIC_API_KEY unset); skipping relevance scoring. Removal/trim proposals still run \u2014 they need no LLM.");let i;try{i=(await w("/api/sessions/project-paths?limit="+ee)).projectPaths}catch(e){n.logger.warn(`Failed to fetch project paths: ${e instanceof Error?e.message:String(e)}`);return}if(i.length===0){n.logger.info("No recent project paths; nothing to score");return}let d=new Map;try{let e=await w("/api/projects?limit=200");for(let t of e.projects??[])t.primaryCheckoutPath&&d.set(x(t.primaryCheckoutPath),t.id)}catch(e){n.logger.warn(`Failed to fetch projects for id resolution: ${e instanceof Error?e.message:String(e)}`)}let g=new Map;try{let e=await w(`/api/catalog?limit=200&proposable=1&contentPreview=${O}`);for(let t of e)g.set(t.id,{id:t.id,name:t.name,description:t.description??"",tags:t.tags??[],contentPreview:(t.contentPreview??"").slice(0,O)})}catch(e){n.logger.warn(`Failed to fetch catalog: ${e instanceof Error?e.message:String(e)}`)}try{let e=await w("/api/instructions?limit=100");for(let t of e.instructions??[])g.has(t.id)||g.set(t.id,{id:t.id,name:t.name,description:t.description??"",tags:t.tags??[],contentPreview:(t.content??"").slice(0,O)})}catch(e){n.logger.warn(`Failed to fetch personal instructions: ${e instanceof Error?e.message:String(e)}`)}let m=[],p=[];try{let e=await w("/api/insights/groom-candidates");if(!e.captureHealthy)n.logger.info(`Grooming paused: ${e.reason??"capture unhealthy"}`);else{m=ce(e),p=ae(e);let t=e.safetyGuarded+e.harmfulSafetyGuarded;n.logger.info(`Removal proposals: ${m.length} (${e.harmful.length} measured-harmful, ${e.candidates.length} never-used, ${t} withheld by the safety allowlist); trim proposals: ${p.length} (${(e.bloated??[]).length} measured-neutral and expensive, ${e.trimSafetyFlagged??0} flagged as containing guardrail language)`)}}catch(e){n.logger.warn(`Failed to fetch groom candidates: ${e instanceof Error?e.message:String(e)}`)}let b=[...g.values()];if(b.length===0&&m.length===0&&p.length===0){n.logger.info("No candidate instructions and nothing to retire; nothing to score");return}n.logger.info(`Scoring ${b.length} candidates across ${i.length} project(s)`);let M=0,l=null,I=new Set;for(let e of i)try{let t=x(e);if(!Q(t)){n.logger.info(`Skipping ${t}: not a project directory (home/root)`);continue}let f=D(t),k=o&&f.length>0;f.length===0&&n.logger.info(`${t}: insufficient language signals; removal/trim only`);let v=X(t),z={name:y.basename(t),description:v.description,languages:f,topics:v.topics,dependencies:v.dependencies},E=k?U(b):[];k&&n.logger.info(`Scoring ${b.length} candidates for ${t} in ${E.length} batch(es)`);let C=[],$=0;for(let r of E)try{let c=await B(H(z,r),"",4096);if(!c){$++;continue}let T=c.recommendations??[];for(let P of T)typeof P.instructionId=="string"&&typeof P.score=="number"&&typeof P.reason=="string"&&C.push({instructionId:P.instructionId,score:P.score,reason:P.reason})}catch(c){$++,n.logger.warn(`Batch failed for ${t}: ${c instanceof Error?c.message:String(c)}`)}$>0&&n.logger.warn(`${$}/${E.length} batch(es) failed for ${t} (backend ${a}); suggestions are partial`),k&&C.length===0&&n.logger.warn(`No usable LLM output for ${t} (backend ${a})`);let K=C.filter(r=>r.score>=ne).sort((r,c)=>c.score-r.score).slice(0,Z),_=d.get(t),W=l===null||l===t;l===null&&(l=t);let N=u(r=>r.topProjectId?r.topProjectId===_:W,"placeHere"),F=m.filter(N),{nominated:L,unmeasurable:G}=await ie(F,t,R);L>0&&n.logger.info(`${t}: ${L} removal candidate(s) nominated for randomised withholding instead of the feed`);let h=[...K.map(r=>({instructionId:r.instructionId,score:r.score,reason:r.reason,kind:"add"})),...G.map(({topProjectId:r,...c})=>({...c,kind:"remove"})),...p.filter(N).map(({estimatedTokens:r,topProjectId:c,...T})=>({...T,kind:"trim"}))];if(h.length===0){n.logger.info(`Nothing to suggest for ${t}`);continue}await R("/api/instruction-suggestions/batch-upsert",{projectPath:t,..._?{projectId:_}:{},suggestions:h}),n.logger.info(`${t}: ${h.filter(r=>r.kind==="add").length} add + ${h.filter(r=>r.kind==="remove").length} remove + ${h.filter(r=>r.kind==="trim").length} trim`);for(let r of h)r.kind!=="add"&&I.add(r.instructionId);let Y=new Set(G.map(r=>r.instructionId));for(let r of F)Y.has(r.instructionId)||I.add(r.instructionId);M++}catch(t){n.logger.warn(`Failed to score ${e}: ${t instanceof Error?t.message:String(t)}`)}let j=[...m.filter(e=>!I.has(e.instructionId)),...p.filter(e=>!I.has(e.instructionId)).map(({estimatedTokens:e,...t})=>t)];if(j.length>0&&l)try{let e=new Set(p.map(t=>t.instructionId));await R("/api/instruction-suggestions/batch-upsert",{projectPath:l,...d.get(l)?{projectId:d.get(l)}:{},suggestions:j.map(({topProjectId:t,...f})=>({...f,kind:e.has(f.instructionId)?"trim":"remove"}))}),n.logger.info(`Orphan sweep: ${j.length} proposal(s) whose project was not scored this run placed on ${l}`)}catch(e){n.logger.warn(`Orphan sweep failed: ${e instanceof Error?e.message:String(e)}`)}let S=le();if(S==="unknown")n.logger.warn("Skills registry unreadable or has no usable generatedAt; leaving suggestion staleness unchanged");else try{await R("/api/instruction-suggestions/mark-stale",{stale:S==="stale"}),S==="stale"?n.logger.warn("Skills registry is stale (>48h); marked catalog-skill suggestions stale"):n.logger.info("Skills registry is fresh; cleared staleness on catalog-skill suggestions")}catch{}n.logger.info(`Done: scored ${M}/${i.length} projects`)},"cronSuggestInstructions");export{Q as a,ae as b,ie as c,ce as d,Re as e};
+import{c as X}from"./chunk-HOQPVOMF.js";import{b as D}from"./chunk-GIRPEMTW.js";import{a as x}from"./chunk-MODXEGCU.js";import{e as J,g as B}from"./chunk-XCH4HPEZ.js";import{e as w,f as R}from"./chunk-GIXHG5FK.js";import{c as H,d as U}from"./chunk-X6CF5KTW.js";import{a as u}from"./chunk-ZLLER3HP.js";import{readFileSync as q,realpathSync as V}from"node:fs";import y from"node:path";import A from"node:os";function Q(s){let n=y.resolve(s);if(n===y.resolve(A.homedir()))return!1;let a=y.parse(n).root;if(n===a)return!1;try{let o=V(A.tmpdir());if(V(n)===o)return!1}catch{}return!0}u(Q,"isScorableProjectPath");var Z=5,ee=Number(process.env.RULEMETRIC_MAX_SUGGESTION_PROJECTS??10),te=48,ne=.5,O=400,oe=3,re=.6,se=1;function ae(s){let n=s.bloated??[],a=new Set([...s.harmful.map(o=>o.id),...s.candidates.map(o=>o.id)]);return n.filter(o=>!a.has(o.id)).slice(0,se).map(o=>({instructionId:o.id,estimatedTokens:o.estimatedTokens,topProjectId:o.topProjectId??null,score:.55,reason:`Costs ~${o.estimatedTokens} tokens every session and measures neutral across ${o.exposures} exposed session(s) \u2014 enough data to detect an effect, and none found. Consider shortening "${o.name}" rather than removing it.`+(o.containsSafetyLanguage?" CAUTION: it contains guardrail language (destructive-command or credential rules) \u2014 preserve those lines when shortening.":"")}))}u(ae,"buildTrimProposals");async function ie(s,n,a){let o=[],i=0;for(let d of s)try{await a("/api/instruction-experiments",{instructionId:d.instructionId,projectPath:n,source:"groom"}),i++}catch(g){let m=g instanceof Error?g.message:String(g);o.push({...d,reason:`${d.reason} (not measurable by withholding: ${m})`})}return{nominated:i,unmeasurable:o}}u(ie,"nominateRemovals");function ce(s){let n=[];for(let o of s.harmful){let i=o.verdict.delta;n.push({instructionId:o.id,score:Math.min(.95,.75+Math.abs(i)),topProjectId:o.topProjectId??null,reason:`Measured harmful: sessions using "${o.name}" scored ${(i*100).toFixed(1)}pp worse across ${o.exposures} exposed session(s). Observational \u2014 review before archiving.`})}let a=new Set(s.harmful.map(o=>o.id));for(let o of s.candidates)a.has(o.id)||n.push({instructionId:o.id,score:re,topProjectId:o.topProjectId??null,reason:`Unused: "${o.name}" is ${o.reason} (${o.totalExposures} lifetime exposure(s)) but still costs context budget every session.`});return n.slice(0,oe)}u(ce,"buildRemovalProposals");function le(){let s=y.join(A.homedir(),".config","rulemetric","skills-registry.json");try{let n=q(s,"utf-8"),a=JSON.parse(n),o=new Date(a.generatedAt).getTime();return Number.isNaN(o)?"unknown":Date.now()-o>te*60*60*1e3?"stale":"fresh"}catch{return"unknown"}}u(le,"observeSkillsRegistry");var Re=u(async(s,n)=>{let a=J(),o=a!=="none";o?n.logger.info(`Scoring backend: ${a}`):n.logger.warn("No LLM backend available (claude CLI missing and ANTHROPIC_API_KEY unset); skipping relevance scoring. Removal/trim proposals still run \u2014 they need no LLM.");let i;try{i=(await w("/api/sessions/project-paths?limit="+ee)).projectPaths}catch(e){n.logger.warn(`Failed to fetch project paths: ${e instanceof Error?e.message:String(e)}`);return}if(i.length===0){n.logger.info("No recent project paths; nothing to score");return}let d=new Map;try{let e=await w("/api/projects?limit=200");for(let t of e.projects??[])t.primaryCheckoutPath&&d.set(x(t.primaryCheckoutPath),t.id)}catch(e){n.logger.warn(`Failed to fetch projects for id resolution: ${e instanceof Error?e.message:String(e)}`)}let g=new Map;try{let e=await w(`/api/catalog?limit=200&proposable=1&contentPreview=${O}`);for(let t of e)g.set(t.id,{id:t.id,name:t.name,description:t.description??"",tags:t.tags??[],contentPreview:(t.contentPreview??"").slice(0,O)})}catch(e){n.logger.warn(`Failed to fetch catalog: ${e instanceof Error?e.message:String(e)}`)}try{let e=await w("/api/instructions?limit=100");for(let t of e.instructions??[])g.has(t.id)||g.set(t.id,{id:t.id,name:t.name,description:t.description??"",tags:t.tags??[],contentPreview:(t.content??"").slice(0,O)})}catch(e){n.logger.warn(`Failed to fetch personal instructions: ${e instanceof Error?e.message:String(e)}`)}let m=[],p=[];try{let e=await w("/api/insights/groom-candidates");if(!e.captureHealthy)n.logger.info(`Grooming paused: ${e.reason??"capture unhealthy"}`);else{m=ce(e),p=ae(e);let t=e.safetyGuarded+e.harmfulSafetyGuarded;n.logger.info(`Removal proposals: ${m.length} (${e.harmful.length} measured-harmful, ${e.candidates.length} never-used, ${t} withheld by the safety allowlist); trim proposals: ${p.length} (${(e.bloated??[]).length} measured-neutral and expensive, ${e.trimSafetyFlagged??0} flagged as containing guardrail language)`)}}catch(e){n.logger.warn(`Failed to fetch groom candidates: ${e instanceof Error?e.message:String(e)}`)}let b=[...g.values()];if(b.length===0&&m.length===0&&p.length===0){n.logger.info("No candidate instructions and nothing to retire; nothing to score");return}n.logger.info(`Scoring ${b.length} candidates across ${i.length} project(s)`);let M=0,l=null,I=new Set;for(let e of i)try{let t=x(e);if(!Q(t)){n.logger.info(`Skipping ${t}: not a project directory (home/root)`);continue}let f=D(t),k=o&&f.length>0;f.length===0&&n.logger.info(`${t}: insufficient language signals; removal/trim only`);let v=X(t),z={name:y.basename(t),description:v.description,languages:f,topics:v.topics,dependencies:v.dependencies},E=k?U(b):[];k&&n.logger.info(`Scoring ${b.length} candidates for ${t} in ${E.length} batch(es)`);let C=[],$=0;for(let r of E)try{let c=await B(H(z,r),"",4096);if(!c){$++;continue}let T=c.recommendations??[];for(let P of T)typeof P.instructionId=="string"&&typeof P.score=="number"&&typeof P.reason=="string"&&C.push({instructionId:P.instructionId,score:P.score,reason:P.reason})}catch(c){$++,n.logger.warn(`Batch failed for ${t}: ${c instanceof Error?c.message:String(c)}`)}$>0&&n.logger.warn(`${$}/${E.length} batch(es) failed for ${t} (backend ${a}); suggestions are partial`),k&&C.length===0&&n.logger.warn(`No usable LLM output for ${t} (backend ${a})`);let K=C.filter(r=>r.score>=ne).sort((r,c)=>c.score-r.score).slice(0,Z),_=d.get(t),W=l===null||l===t;l===null&&(l=t);let N=u(r=>r.topProjectId?r.topProjectId===_:W,"placeHere"),F=m.filter(N),{nominated:L,unmeasurable:G}=await ie(F,t,R);L>0&&n.logger.info(`${t}: ${L} removal candidate(s) nominated for randomised withholding instead of the feed`);let h=[...K.map(r=>({instructionId:r.instructionId,score:r.score,reason:r.reason,kind:"add"})),...G.map(({topProjectId:r,...c})=>({...c,kind:"remove"})),...p.filter(N).map(({estimatedTokens:r,topProjectId:c,...T})=>({...T,kind:"trim"}))];if(h.length===0){n.logger.info(`Nothing to suggest for ${t}`);continue}await R("/api/instruction-suggestions/batch-upsert",{projectPath:t,..._?{projectId:_}:{},suggestions:h}),n.logger.info(`${t}: ${h.filter(r=>r.kind==="add").length} add + ${h.filter(r=>r.kind==="remove").length} remove + ${h.filter(r=>r.kind==="trim").length} trim`);for(let r of h)r.kind!=="add"&&I.add(r.instructionId);let Y=new Set(G.map(r=>r.instructionId));for(let r of F)Y.has(r.instructionId)||I.add(r.instructionId);M++}catch(t){n.logger.warn(`Failed to score ${e}: ${t instanceof Error?t.message:String(t)}`)}let j=[...m.filter(e=>!I.has(e.instructionId)),...p.filter(e=>!I.has(e.instructionId)).map(({estimatedTokens:e,...t})=>t)];if(j.length>0&&l)try{let e=new Set(p.map(t=>t.instructionId));await R("/api/instruction-suggestions/batch-upsert",{projectPath:l,...d.get(l)?{projectId:d.get(l)}:{},suggestions:j.map(({topProjectId:t,...f})=>({...f,kind:e.has(f.instructionId)?"trim":"remove"}))}),n.logger.info(`Orphan sweep: ${j.length} proposal(s) whose project was not scored this run placed on ${l}`)}catch(e){n.logger.warn(`Orphan sweep failed: ${e instanceof Error?e.message:String(e)}`)}let S=le();if(S==="unknown")n.logger.warn("Skills registry unreadable or has no usable generatedAt; leaving suggestion staleness unchanged");else try{await R("/api/instruction-suggestions/mark-stale",{stale:S==="stale"}),S==="stale"?n.logger.warn("Skills registry is stale (>48h); marked catalog-skill suggestions stale"):n.logger.info("Skills registry is fresh; cleared staleness on catalog-skill suggestions")}catch{}n.logger.info(`Done: scored ${M}/${i.length} projects`)},"cronSuggestInstructions");export{Q as a,ae as b,ie as c,ce as d,Re as e};

package/dist/chunk-CMDOLJ23.js

@@ -0,0 +1 @@
+import{g}from"./chunk-HT6R6KR3.js";import{e as c,f as p}from"./chunk-GIXHG5FK.js";import{a}from"./chunk-ZLLER3HP.js";import{createHash as C,randomUUID as R}from"node:crypto";import{lstatSync as u,readFileSync as b,writeFileSync as v,openSync as k,closeSync as y,fsyncSync as x,renameSync as $,unlinkSync as A}from"node:fs";import{isAbsolute as I,parse as F,join as E,dirname as P}from"node:path";var w=a(e=>C("sha256").update(e).digest("hex"),"hash");function f(e){if(!I(e))throw new Error("Restoration requires an absolute file anchor");let o=F(e).root;for(let t of e.slice(o.length).split("/"))if(o=E(o,t),u(o).isSymbolicLink())throw new Error("Restoration refuses symlink anchors");if(!u(e).isFile())throw new Error("Restoration requires a regular file");return b(e,"utf8")}a(f,"readFile");function h(e,o,t){if(f(e)!==o)return!1;let i=E(P(e),`.rulemetric-restore-${R()}`),n;try{return n=k(i,"wx",u(e).mode&511),v(n,t),x(n),y(n),n=void 0,f(e)!==o?!1:($(i,e),!0)}finally{n!==void 0&&y(n);try{A(i)}catch(s){if(s.code!=="ENOENT")throw s}}}a(h,"replaceExact");async function j(e,o){let t=await c(`/api/promotions/${o.promotionId}`),i=await c(`/api/eval-targets/${e}`),n=a(r=>({applied:!1,fileWritten:!1,reason:r}),"refuse");if(o.version!==1||o.authority!=="manual"||t.evalTargetId!==e||!t.applied||t.decision!=="promote"||i.content!==t.toContent)return n("Stale or unsupported restoration");let s=i.metadata?.immutableSections;if(!Array.isArray(s)||!s.every(r=>typeof r=="string"))return n("Explicit immutable sections are required");let S=Array.isArray(t.gates?.immutableSections)?t.gates.immutableSections.filter(r=>typeof r=="string"):[];if(g(t.toContent,t.fromContent,[...s,...S]).length)return n("Restoration would change immutable sections");let d=!1;try{if(!h(i.name,t.toContent,t.fromContent))return n("File changed; preserving human edits");if(d=!0,f(i.name)!==t.fromContent)throw new Error("Predecessor read-back failed");let r=await p(`/api/promotions/${t.id}/restore`,{authority:o.authority,reason:o.reason,expectedVersion:i.version,path:i.name,diskBeforeHash:w(t.toContent),diskAfterHash:w(t.fromContent)});return{applied:r.applied,fileWritten:!0,promotionId:r.id}}catch(r){if(!d)return n(r instanceof Error?r.message:String(r));let l=await c(`/api/eval-targets/${e}`).catch(()=>null),m=await c(`/api/promotions/${t.id}/restoration`).catch(()=>null);if(m&&l?.content===t.fromContent&&l.version===m.gates.newVersion&&f(i.name)===t.fromContent)return{applied:!0,fileWritten:!0,promotionId:m.id,reason:"Restoration verified after response failure"};if(!l||l.content===t.fromContent)throw new Error("Restoration state unknown; disk retained for reconciliation",{cause:r});return h(i.name,t.fromContent,t.toContent),n(`Restoration refused: ${r instanceof Error?r.message:String(r)}`)}}a(j,"restoreInstruction");export{j as a};

package/dist/chunk-DYET4EYU.js

@@ -0,0 +1 @@
+import{a as _e}from"./chunk-M7QSC64D.js";import{i as Me}from"./chunk-7LFCZI64.js";import{a as Pe}from"./chunk-CMDOLJ23.js";import{b as pe,i as me,j as ge,k as fe,l as he,m as be,n as $e}from"./chunk-TWPD3ICV.js";import{c as Re}from"./chunk-G7V4ZZZ3.js";import{a as xe,b as Se,c as De,d as Ie,e as se,f as Oe}from"./chunk-LJE5J3EW.js";import{a as re}from"./chunk-AHLVY52I.js";import{d as ke,f as Ce,g as Ee,i as Ae}from"./chunk-YXMAGC53.js";import{a as we}from"./chunk-T7HZZ4DJ.js";import{c as ye}from"./chunk-FADINV4H.js";import{b as ve}from"./chunk-SWSOEYC7.js";import{d as ue,e as ne,f as de}from"./chunk-HT6R6KR3.js";import{e as y,f as te}from"./chunk-GIXHG5FK.js";import{a as g}from"./chunk-ZLLER3HP.js";import{createHash as Ge}from"node:crypto";import{existsSync as Te,readFileSync as Ne,writeFileSync as Xe}from"node:fs";var H=g(t=>Ge("sha256").update(t).digest("hex"),"sha256"),Le=1.3,ae=4,ze=20;function Be(t,o){if(!t.configured||o<=0)return ae;let e=Math.ceil(t.minPairs*Le/o);return Math.min(ze,Math.max(ae,e))}g(Be,"passCapFor");function Qe(t,o=10,e=ae){if(t<=0)return{passes:0,expectedPairs:0,feasible:!1,reason:"no live cases"};let i=Math.ceil(o*Le),l=Math.min(e,Math.ceil(i/t)),r=l*t;return r<o?{passes:l,expectedPairs:r,feasible:!1,reason:`${t} live case(s) x ${e} passes = ${t*e} pairs at best, below the ${o} a controlled verdict requires \u2014 add cases rather than replaying these`}:{passes:l,expectedPairs:r,feasible:!0}}g(Qe,"planPasses");var Ve=60,Ye=5,We=g(t=>Array.isArray(t)?t.filter(o=>typeof o=="string"):[],"stringList"),J=g(t=>typeof t=="number"&&Number.isFinite(t)?t:null,"finite");function Ze(t,o,e=Ye){let i=[...t].filter(s=>s.fromContentHash===o).sort((s,c)=>Date.parse(c.createdAt)-Date.parse(s.createdAt)),l=[],r=new Set;for(let s of i){let c=s.toContentHash??s.id;if(r.has(c))continue;r.add(c);let d=s.verdict??null,k=J(d?.delta),M=J(d?.ci?.lower),x=J(d?.ci?.upper),S=J(d?.nWith),L=d?.sufficient===!0&&k!=null&&M!=null&&x!=null&&S!=null?{delta:k,ciLower:M,ciUpper:x,nPairs:S}:null;if(l.push({decision:typeof s.decision=="string"?s.decision:"unknown",applied:s.applied===!0,at:typeof s.createdAt=="string"?s.createdAt:"",rationale:typeof s.proposalRationale=="string"?s.proposalRationale:"",touched:We(s.touchedSections),refusals:We(s.refusals),measured:L}),l.length>=e)break}return l}g(Ze,"summarisePriorDecisions");function et(t,o){let e=o.accumulate??(t.autoEvolveAccumulate===!0||t.autoEvolveAccumulate==="true"),i=o.accumulateTargetPairs??t.autoEvolveAccumulateTarget,l=typeof i=="string"?Number.parseInt(i,10):i,r=typeof l=="number"&&Number.isInteger(l)&&l>0?l:Ve;return{enabled:e,targetPairs:r}}g(et,"readAccumulateConfig");function tt(t){let o=t.accumulateEnabled&&t.pairCount<t.targetPairs;return{apply:t.consented&&!o,peeking:o}}g(tt,"resolveApply");async function Pt(t,o){if(t.restoration)return Pe(t.evalTargetId,t.restoration);let e=g(n=>o?.logger?.info?.(n)??console.log(n),"log"),{evalTargetId:i}=t,l=t.measurement?ye(t):{},r=`[cron_instruction_evolution] ${i}`,s=await y(`/api/eval-targets/${i}`),c=s.metadata??{},d=t.measurement?JSON.stringify([l.engine,l.model,l.graderEngine,l.graderModel,s.type,ve(t.measurement.engine)]):void 0,k=null,M=!1;if(s.name.startsWith("/")&&Te(s.name))try{if(k=Ne(s.name,"utf8"),k!==s.content){let n=await te(`/api/eval-targets/${i}/sync-content`,{content:k});M=n.synced,e(`${r}: target content drifted from disk by ${Math.abs(k.length-s.content.length)} chars \u2014 synced to disk truth (v${n.version}). Prior verdicts against the old text dissolve via content-hash keying.`),s={...s,content:k,version:n.version}}}catch(n){e(`${r}: disk sync failed (${n instanceof Error?n.message:String(n)}) \u2014 measuring the stored snapshot`)}let x=!1;try{x=(await y("/api/settings/evolution-account")).paused}catch(n){e(`${r}: could not read the account switch (${n instanceof Error?n.message:String(n)}) \u2014 pausing to be safe`),x=!0}if(x)return e(`${r}: PAUSED for the whole account (profiles.evolution_paused) \u2014 no model budget spent.`),{skipped:"paused-account"};if(ge(c))return e(`${r}: PAUSED (metadata.autoEvolvePaused) \u2014 no model budget spent. Opt-in is intact.`),{skipped:"paused"};let S=_e(s);if(S)return e(`${r}: ${S.reason}`),S;let L=t.apply??(c.autoEvolveApply===!0||c.autoEvolveApply==="true"),F=nt(c);if(F==null){if(L)return e(`${r}: refusing \u2014 metadata.immutableSections is unset. Set it (use [] to explicitly protect nothing) before letting the loop edit this target unattended.`),{skipped:"immutable-sections-unset"};e(`${r}: metadata.immutableSections unset \u2014 measure-only cycle proceeds with [] (nothing protected). Writing still requires setting it explicitly.`),F=[]}let v=(await y(`/api/eval-targets/${i}/evals`)).filter(n=>!n.retiredAt);if(v.length===0)return{skipped:"no-live-cases"};let j=me(c);j.note&&e(`${r}: ${j.note}`);let C=Qe(v.length,j.minPairs,Be(j,v.length));if(!C.feasible)return e(`${r}: cannot reach a sufficient verdict \u2014 ${C.reason}`),{skipped:"cannot-reach-min-n",reason:C.reason};let Fe=fe({liveCases:v.length,passes:C.passes});e(`${r}: budget for this run \u2014 ${$e(Fe)}`);let G=await y(`/api/eval-targets/${i}/judge-calibration`);if(!G.powerCheck.adequate){let n=(G.powerCheck.reasons??[]).join("; ");return e(`${r}: not adequately powered, skipping before spending budget \u2014 ${n}`),{skipped:"not-adequate",reason:n}}let oe=de,je=Date.now()-oe*24*60*60*1e3,X=await y(`/api/eval-targets/${i}/promotions?limit=200`),ie=X.promotions.filter(n=>n.applied&&Date.parse(n.createdAt)>=je).length;if(ie>=ne)return e(`${r}: change budget looks exhausted (${ie}/${ne} applied in ${oe}d), skipping before spending budget`),{skipped:"budget-exhausted"};try{let n=await y("/api/usage/current-limits"),u=Re({limits:n,now:new Date});if(!u.ok)return e(`${r}: deferring before spending budget \u2014 ${u.reason}`),{skipped:"quota-exhausted",reason:u.reason}}catch(n){e(`${r}: quota unknown (${n instanceof Error?n.message:String(n)}) \u2014 proceeding`)}let U=ue();if(!U.green)return e(`${r}: A/A canary is RED \u2014 halting. ${U.failures.join(" | ")}`),{skipped:"canary-red",reason:U.failures.join(" | ")};let z=s.content,D=H(z),f=et(c,t),w=f.enabled?re(X.promotions,D,f.targetPairs,d):null,b,q,$=[],_=0,B=pe,p=w?null:Se(c,D,d);if(p){b=p.content,q="",B=0;try{$=await Ie({evalTargetId:i,candidateContentHash:p.hash,executionKey:d,observedKey:p.observedKey})}catch(n){return e(`${r}: could not rebuild banked pairs (${n instanceof Error?n.message:String(n)})`),{skipped:"resume-read-failed",reason:p.hash}}e(`${r}: resuming interrupted cycle \u2014 ${$.length}/${p.plannedPairs} pair(s) rebuilt from eval_runs, no proposal call spent`)}else if(w){let n=await y(`/api/promotions/${w.promotionId}`);if(H(n.toContent)!==w.candidateContentHash)return e(`${r}: stored candidate does not match its recorded hash \u2014 refusing to continue`),{skipped:"accumulation-hash-mismatch",reason:w.promotionId};b=n.toContent,q=n.proposalRationale??"",$=w.pairs,_=w.runs,B=0,e(`${r}: continuing accumulation ${w.promotionId} \u2014 ${$.length}/${f.targetPairs} pair(s) carried from ${_} prior run(s)`)}else{let n=G.caseDiscrimination?.cases??[],u=[];if(s.name.startsWith("/")){let a=s.name.slice(0,s.name.lastIndexOf("/"));try{let m=await y(`/api/insights/evidence?projectPath=${encodeURIComponent(a)}`);m.analyzedAt&&(u=[...m.friction.map(ee=>`Real-session friction (insights, ${m.analyzedAt.slice(0,10)}): ${ee}`),...m.whatWorks.map(ee=>`Working well in real sessions \u2014 do not break: ${ee}`)],u.length>0&&e(`${r}: proposer sees ${m.friction.length} friction + ${m.whatWorks.length} what-works item(s) from insights`))}catch(m){e(`${r}: insights evidence unavailable (${m instanceof Error?m.message:String(m)}) \u2014 proposing without it`)}}let N=[];try{N=Ze(X.promotions,D)}catch(a){e(`${r}: could not summarise prior decisions (${a instanceof Error?a.message:String(a)}) \u2014 proposing without them`)}N.length>0&&e(`${r}: proposer sees ${N.length} prior decision(s) against this exact body (${N.map(a=>`${a.decision}${a.measured?"/measured":""}`).join(", ")})`);let W=await Me({currentContent:s.content,targetType:s.type,targetName:s.name,immutableSections:F,evidence:{nonDiscriminating:n.filter(a=>!a.discriminating).map(a=>`${a.name??a.assertion??"unnamed"} \u2014 ${a.note}`),weakest:n.filter(a=>a.discriminating&&a.withPassRate<a.withoutPassRate).map(a=>({name:a.name??a.assertion??"unnamed",withPassRate:a.withPassRate,withoutPassRate:a.withoutPassRate})),priorDecisions:N,notes:u}},void 0,t.measurement?(a,m)=>we(t.measurement,a,m):void 0);if(!W.ok)return e(`${r}: no usable proposal \u2014 ${W.reason}`),{skipped:"no-proposal",reason:W.reason};b=W.proposal.candidateContent,q=W.proposal.rationale}let I=p?De({plannedPairs:p.plannedPairs||C.expectedPairs,carriedPairs:$.length,liveCases:v.length}):C.passes;e(`${r}: measuring ${v.length} case(s) x ${I} pass(es) = up to ${v.length*I} pairs, candidate vs prior version`+(p?` (resumed; ${$.length} pair(s) already banked)`:""));let E=[],K=[],Ue=Date.now(),le=0,Q=0;for(let n=1;n<=I;n++){let u=await Ae({cases:v,candidateContent:b,priorVersionContent:z,concurrency:t.concurrency??2,runNumber:n,opts:{targetId:i,targetVersion:s.version??1,iteration:1,timeout:t.timeout??24e4,model:t.model,graderModel:t.graderModel,graderEngine:t.graderEngine,...l,executionKey:d,skipGrading:!1,targetType:s.type,targetName:s.name,candidateContentHash:H(b)},onLog:e});if(E.push(...u.pairs),K.push(...u.results),le+=u.completed,Q+=u.failed,e(`${r}: pass ${n}/${I} \u2014 ${u.pairs.length} pair(s), running total ${E.length}`),u.abortedForQuota){e(`${r}: stopping after pass ${n}/${I} \u2014 ${u.abortedForQuota}. Remaining passes would spend against an exhausted account.`);break}}let O=ke(K),qe=p?.observedKey??w?.observedKey;if(t.measurement&&E.length>0&&(!O||$.length>0&&qe!==O))return{skipped:"incomparable-execution",reason:"Actual actor/judge models or delivery changed; recorded runs are retained, but incompatible evidence cannot be pooled or promoted."};let A=[...$,...E],T=Ce(K),V=E.length>0,h=he({runs:K,plannedPairs:C.expectedPairs,usablePairs:E.length,verdictPairs:V?A.length:E.length,proposalCalls:B,passes:I,liveCases:v.length,wallMs:Date.now()-Ue});if(e(`${r}: ${le} completed, ${Q} failed, ${E.length} usable pair(s) \u2014 spent ~${h.modelCalls} model calls (${h.proposalCalls} proposal + ${h.executorCalls} executor + >=${h.graderCalls} grader) in ${(h.wallMs/6e4).toFixed(1)} min`+(V?`; the ${h.verdictPairs} pair(s) behind this verdict resolve +/-${be(h.mde)} at 80% power`:"")),!V)return{skipped:T.infrastructureFailed?"infrastructure-failure":"no-pairs",reason:T.infrastructureFailed?`${T.infrastructureFailed} run(s) never executed (${T.summary.describe()}) \u2014 the environment failed, not the target`:`${Q} run(s) failed or were ungradeable`,cost:h};let Y=Ee({pairs:A.length,minPairs:10,accounting:T});if(Y.kind==="infrastructure-incomplete")return e(`${r}: not posting a verdict \u2014 ${Y.reason}`),await se({evalTargetId:i,meta:c,blob:Oe({executionKey:d,...O?{observedKey:O}:{},content:b,hash:H(b),priorHash:D,plannedPairs:p?.plannedPairs||C.expectedPairs,now:new Date}),log:e,tag:r}),{skipped:"infrastructure-incomplete",reason:Y.reason,cost:h};f.enabled&&e(`${r}: accumulated ${A.length}/${f.targetPairs} pair(s) over ${_+1} run(s) on one candidate`);let ce=L,{apply:Ke,peeking:Je}=tt({consented:ce,accumulateEnabled:f.enabled,pairCount:A.length,targetPairs:f.targetPairs});ce&&Je&&e(`${r}: interim readout at ${A.length}/${f.targetPairs} pairs \u2014 recording the decision but not acting until the target is reached`);let P=await te(`/api/eval-targets/${i}/promotions`,{candidateContent:b,proposalSource:"llm",proposalRationale:q,pairs:A,canaryGreen:U.green,measuredAgainstContentHash:D,immutableSections:F,apply:Ke,measurementCost:h,...f.enabled?{accumulation:{executionKey:d,...O?{observedKey:O}:{},candidateContentHash:H(b),priorContentHash:D,targetPairs:f.targetPairs,carriedPairs:$.length,runs:_+1}}:{}});e(`${r}: ${P.decision}${P.applied?" (APPLIED)":""} \u2014 ${P.rationale}`+(P.refusals.length>0?` | refusals: ${P.refusals.join("; ")}`:"")),(p||c[xe])&&await se({evalTargetId:i,meta:c,blob:null,log:e,tag:r});let Z=!1,R;if(P.applied&&s.name.startsWith("/"))try{Te(s.name)?Ne(s.name,"utf8")!==z?R="file changed since measurement started \u2014 refusing to clobber; next cycle re-syncs":(Xe(s.name,b,"utf8"),Z=!0):R="file no longer exists on disk",e(Z?`${r}: measured improvement written to ${s.name} (undo: git checkout -- ${s.name})`:`${r}: NOT written to disk \u2014 ${R}`)}catch(n){R=n instanceof Error?n.message:String(n),e(`${r}: disk write-back failed \u2014 ${R}. The eval target is updated; the file is not.`)}return{decision:P.decision,applied:P.applied,nPairs:A.length,promotionId:P.id,cost:h,diskSynced:M,...P.applied?{fileWritten:Z,...R?{fileWriteSkipped:R}:{}}:{},...f.enabled?{carriedPairs:$.length,accumulationRuns:_+1}:{}}}g(Pt,"cronInstructionEvolution");function nt(t){let o=t.immutableSections;return o==null||!Array.isArray(o)?null:o.filter(e=>typeof e=="string")}g(nt,"readImmutableSections");export{Le as a,ae as b,ze as c,Be as d,Qe as e,Ve as f,Ye as g,Ze as h,et as i,tt as j,Pt as k,nt as l};

package/dist/{chunk-RR7V5CK5.js → chunk-EJY5GK6A.js}

@@ -1 +1 @@
-import{b as m}from"./chunk-WIKCYUZC.js";import{a as g}from"./chunk-ZLLER3HP.js";var p=5;function b(t){let r=g((e,f)=>({retire:[],survivingCases:0,replacementsRequired:!1,refused:e,summary:f}),"none");if(!t||t.cases.length===0&&t.unpairedCases.length===0)return r("no-discrimination-data","No case-discrimination data yet \u2014 the target needs runs in BOTH cohorts before any case can be judged dead.");let c=[...t.cases.filter(e=>!e.discriminating).map(e=>({evalId:e.evalId,name:e.name,feedback:{name:e.name,withPassRate:e.withPassRate,withoutPassRate:e.withoutPassRate,reason:e.note}})),...t.unpairedCases.map(e=>({evalId:e.evalId,name:e.name,feedback:{name:e.name,withPassRate:e.withPassRate??0,withoutPassRate:e.withoutPassRate,reason:"runs in only one cohort (no with-vs-without comparison possible)"}}))];if(c.length===0)return r("all-discriminating","Every case discriminates \u2014 nothing to retire.");let s=new Map(t.cases.map(e=>[e.evalId,e.separation])),o=[...c].sort((e,f)=>(s.get(e.evalId)??-1)-(s.get(f.evalId)??-1)).slice(0,p),u=c.length-o.length,d=t.total+t.unpaired,l=d-o.length,i=l<m,n=`${o.length} of ${d} live case(s) carry no treatment signal`+(u>0?` (${u} more over the ${p}-per-run cap)`:"")+(i?`; retiring them leaves ${l}, below the ${m}-case evolution floor, so replacements must be generated first`:"");return{retire:o,survivingCases:l,replacementsRequired:i,refused:null,summary:n}}g(b,"planCaseRefinement");async function v(t,r){let h=r.log??(()=>{}),c=g((n,e)=>({retired:0,generated:0,skipped:n,summary:e,retiredNames:[],generatedNames:[]}),"empty");if(t.refused)return c(t.refused,t.summary);let s=await r.generate(t.retire.map(n=>n.feedback)),a=[];for(let n of s??[]){let e=await r.createCase(n);a.push(e.name),h(`  + ${e.name} (${e.id})`)}let o=t.survivingCases+t.retire.length+a.length,u=Math.max(0,o-m),d=t.retire.slice(0,Math.min(t.retire.length,u)),l=t.retire.length-d.length;if(d.length===0)return{retired:0,generated:a.length,skipped:s===null?"no-llm-backend":"no-replacements-generated",summary:`${t.summary}. `+(s===null?"No local LLM backend available, so nothing was retired":`The generator produced ${a.length} usable replacement(s), not enough to retire anything`)+` \u2014 dropping below the ${m}-case floor would silently disenrol the target from the nightly loop.`,retiredNames:[],generatedNames:a};let i=[];for(let n of d)await r.retireCase(n.evalId,n.feedback.reason),i.push(n.name),h(`  retired ${n.name}`);return{retired:i.length,generated:a.length,skipped:null,summary:`Retired ${i.length} non-discriminating case(s), added ${a.length} replacement(s)`+(l>0?`; held ${l} back to stay at the ${m}-case floor (they remain condemned and will be retired once replacements exist)`:"")+". The replacements have no runs yet \u2014 the next autorun batch builds their depth.",retiredNames:i,generatedNames:a}}g(v,"executeCaseRefinement");export{p as a,b,v as c};
+import{b as m}from"./chunk-74KE4VOX.js";import{a as g}from"./chunk-ZLLER3HP.js";var p=5;function b(t){let r=g((e,f)=>({retire:[],survivingCases:0,replacementsRequired:!1,refused:e,summary:f}),"none");if(!t||t.cases.length===0&&t.unpairedCases.length===0)return r("no-discrimination-data","No case-discrimination data yet \u2014 the target needs runs in BOTH cohorts before any case can be judged dead.");let c=[...t.cases.filter(e=>!e.discriminating).map(e=>({evalId:e.evalId,name:e.name,feedback:{name:e.name,withPassRate:e.withPassRate,withoutPassRate:e.withoutPassRate,reason:e.note}})),...t.unpairedCases.map(e=>({evalId:e.evalId,name:e.name,feedback:{name:e.name,withPassRate:e.withPassRate??0,withoutPassRate:e.withoutPassRate,reason:"runs in only one cohort (no with-vs-without comparison possible)"}}))];if(c.length===0)return r("all-discriminating","Every case discriminates \u2014 nothing to retire.");let s=new Map(t.cases.map(e=>[e.evalId,e.separation])),o=[...c].sort((e,f)=>(s.get(e.evalId)??-1)-(s.get(f.evalId)??-1)).slice(0,p),u=c.length-o.length,d=t.total+t.unpaired,l=d-o.length,i=l<m,n=`${o.length} of ${d} live case(s) carry no treatment signal`+(u>0?` (${u} more over the ${p}-per-run cap)`:"")+(i?`; retiring them leaves ${l}, below the ${m}-case evolution floor, so replacements must be generated first`:"");return{retire:o,survivingCases:l,replacementsRequired:i,refused:null,summary:n}}g(b,"planCaseRefinement");async function v(t,r){let h=r.log??(()=>{}),c=g((n,e)=>({retired:0,generated:0,skipped:n,summary:e,retiredNames:[],generatedNames:[]}),"empty");if(t.refused)return c(t.refused,t.summary);let s=await r.generate(t.retire.map(n=>n.feedback)),a=[];for(let n of s??[]){let e=await r.createCase(n);a.push(e.name),h(`  + ${e.name} (${e.id})`)}let o=t.survivingCases+t.retire.length+a.length,u=Math.max(0,o-m),d=t.retire.slice(0,Math.min(t.retire.length,u)),l=t.retire.length-d.length;if(d.length===0)return{retired:0,generated:a.length,skipped:s===null?"no-llm-backend":"no-replacements-generated",summary:`${t.summary}. `+(s===null?"No local LLM backend available, so nothing was retired":`The generator produced ${a.length} usable replacement(s), not enough to retire anything`)+` \u2014 dropping below the ${m}-case floor would silently disenrol the target from the nightly loop.`,retiredNames:[],generatedNames:a};let i=[];for(let n of d)await r.retireCase(n.evalId,n.feedback.reason),i.push(n.name),h(`  retired ${n.name}`);return{retired:i.length,generated:a.length,skipped:null,summary:`Retired ${i.length} non-discriminating case(s), added ${a.length} replacement(s)`+(l>0?`; held ${l} back to stay at the ${m}-case floor (they remain condemned and will be retired once replacements exist)`:"")+". The replacements have no runs yet \u2014 the next autorun batch builds their depth.",retiredNames:i,generatedNames:a}}g(v,"executeCaseRefinement");export{p as a,b,v as c};

package/dist/{chunk-T47SFA5X.js → chunk-FADINV4H.js}

@@ -1 +1 @@
-import{f as w}from"./chunk-SWSOEYC7.js";import{a as C}from"./chunk-KGURGIZ2.js";import{a as M}from"./chunk-DVU5PHPM.js";import{a as x,f as b}from"./chunk-FBSZ6KCM.js";import{d as k}from"./chunk-GPBN4ROE.js";import{f as v}from"./chunk-GIXHG5FK.js";import{f as y}from"./chunk-XQYAVSUQ.js";import{a as m}from"./chunk-ZLLER3HP.js";function $(t,e={}){let i=e.engine??(e.model||t.engine==="pi"?"claude":t.engine);return{engine:i,model:e.model??(i===t.engine?t.model:void 0)}}m($,"selectEvalJudge");function _(t){let e=$(t.actor,t.pin??{});if(!t.available.includes(e.engine))throw new Error(`configured grader unavailable: ${e.engine}`);if(e.engine==="codex"&&!e.model?.trim())throw new Error("Codex grader requires a model pin");return e}m(_,"resolveMeasurementJudge");function A(t,e){let i=t.measurement?y.parse(t.measurement):void 0,r=i?.engine??x(t.engine),o=i?i.model??void 0:t.model;if(!e){let s=k();e=[...s.claudeBin?["claude"]:[],...s.codexBin?["codex"]:[],...s.piBin?["pi"]:[]]}if(!e.includes(r))throw new Error(`configured actor unavailable: ${r}`);let u=t.skipGrading?$({engine:r,model:o}):_({actor:{engine:r,model:o},pin:t.graderEngine||t.graderModel?{engine:t.graderEngine,model:t.graderModel}:null,available:e});return{engine:r,model:o,graderEngine:u.engine,graderModel:u.model,...i?{measurement:i}:{}}}m(A,"resolveMeasurementJob");async function R(t,e,i=()=>{}){let{eval_:r,configuration:o,targetContent:u,runNumber:s}=t;i(`[${o}] ${r.name} run#${s} \u2014 executing...`);let l=x(e.engine),g=w({targetType:e.targetType,targetContent:u,targetName:e.targetName,engine:l}),c={...e.executionKey?{executionKey:e.executionKey}:{},deliveryRegime:`${e.targetType??"prompt"}:${l}:${g.methodology}`,...e.measurement?{measurement:e.measurement}:{},methodology:g.methodology,delivery:g.delivery,isolated:g.isWorktree,...e.candidateContentHash?{candidateContentHash:e.candidateContentHash}:{},...g.leakedFiles.length?{leakedFiles:g.leakedFiles}:{}};g.isWorktree||i(`[${o}] ${r.name} run#${s} \u2014 \u26A0 no git worktree; running without repo context`),g.leakedFiles.length&&i(`[${o}] ${r.name} run#${s} \u2014 \u26A0 CONTAMINATED: ${g.leakedFiles.join(", ")} survived the strip`);try{let n=await b({prompt:r.prompt,targetContent:g.systemPrompt,workDir:g.workDir,timeout:e.timeout,model:e.model,engine:l});if(n.error){let d=C(n.error);return i(`[${o}] ${r.name} run#${s} \u2014 FAILED (${d.kind}): ${n.error}`),{evalId:r.id,evalTargetId:e.targetId,evalTargetVersion:e.targetVersion,configuration:o,runNumber:s,iteration:e.iteration,status:"failed",outputs:{error:n.error,failureKind:d.kind,failureClass:d.class,...c,engine:n.engine??l,model:n.model??null,...n.provider?{provider:n.provider}:{},...n.costUsd!=null?{costUsd:n.costUsd}:{}},timing:{duration_ms:n.durationMs,total_tokens:n.totalTokens},grading:{},failure:d}}i(`[${o}] ${r.name} run#${s} \u2014 completed in ${(n.durationMs/1e3).toFixed(1)}s`);let a={};if(!e.skipGrading&&r.expectations.length>0){i(`[${o}] ${r.name} run#${s} \u2014 grading...`);let d=await M({prompt:r.prompt,output:n.output,expectations:r.expectations,...$({engine:l,model:e.model},{engine:e.graderEngine,model:e.graderModel}),timeout:e.timeout});a=d,i(`[${o}] ${r.name} run#${s} \u2014 grading done: pass_rate=${d.summary.pass_rate==null?"n/a (all assertions errored)":`${(d.summary.pass_rate*100).toFixed(0)}%`}`)}return{evalId:r.id,evalTargetId:e.targetId,evalTargetVersion:e.targetVersion,configuration:o,runNumber:s,iteration:e.iteration,status:"completed",outputs:{transcript:n.output,...c,engine:n.engine??l,...n.provider?{provider:n.provider}:{},model:n.model??null,...n.costUsd!=null?{costUsd:n.costUsd}:{}},timing:{duration_ms:n.durationMs,total_tokens:n.totalTokens,total_duration_seconds:n.durationMs/1e3},grading:a}}finally{g.cleanup()}}m(R,"executeAndGradeCase");async function B(t){let{cases:e,targetContent:i,concurrency:r,runNumber:o,opts:u,skipBaseline:s,onLog:l=m(()=>{},"onLog")}=t,g=s?["with_target"]:["with_target","without_target"],c=[];for(let a of e)for(let d of g)c.push({eval_:a,configuration:d,targetContent:d==="with_target"?i:void 0,runNumber:o});let n=[];for(let a=0;a<c.length;a+=Math.max(1,r)){let d=c.slice(a,a+Math.max(1,r)),h=await Promise.all(d.map(f=>R(f,u,l)));n.push(...h);let p=[];for(let f of h)try{await v("/api/eval-runs",f)}catch(E){p.push(E instanceof Error?E.message:String(E))}if(p.length)throw new Error(`Could not persist ${p.length} eval run(s); stopped further execution: ${p.join("; ")}`)}try{await v(`/api/eval-targets/${u.targetId}/label-state`,{})}catch(a){l(`  \u26A0 label-state notification skipped: ${a instanceof Error?a.message:String(a)}`)}return{completed:n.filter(a=>a.status==="completed").length,failed:n.filter(a=>a.status==="failed").length,results:n}}m(B,"runBatchAndPost");export{$ as a,_ as b,A as c,R as d,B as e};
+import{f as w}from"./chunk-SWSOEYC7.js";import{a as C}from"./chunk-KGURGIZ2.js";import{a as M}from"./chunk-FG343JM3.js";import{a as x,f as b}from"./chunk-4GOXEEXZ.js";import{d as k}from"./chunk-GPBN4ROE.js";import{f as v}from"./chunk-GIXHG5FK.js";import{f as y}from"./chunk-XQYAVSUQ.js";import{a as m}from"./chunk-ZLLER3HP.js";function $(t,e={}){let i=e.engine??(e.model||t.engine==="pi"?"claude":t.engine);return{engine:i,model:e.model??(i===t.engine?t.model:void 0)}}m($,"selectEvalJudge");function _(t){let e=$(t.actor,t.pin??{});if(!t.available.includes(e.engine))throw new Error(`configured grader unavailable: ${e.engine}`);if(e.engine==="codex"&&!e.model?.trim())throw new Error("Codex grader requires a model pin");return e}m(_,"resolveMeasurementJudge");function A(t,e){let i=t.measurement?y.parse(t.measurement):void 0,r=i?.engine??x(t.engine),o=i?i.model??void 0:t.model;if(!e){let s=k();e=[...s.claudeBin?["claude"]:[],...s.codexBin?["codex"]:[],...s.piBin?["pi"]:[]]}if(!e.includes(r))throw new Error(`configured actor unavailable: ${r}`);let u=t.skipGrading?$({engine:r,model:o}):_({actor:{engine:r,model:o},pin:t.graderEngine||t.graderModel?{engine:t.graderEngine,model:t.graderModel}:null,available:e});return{engine:r,model:o,graderEngine:u.engine,graderModel:u.model,...i?{measurement:i}:{}}}m(A,"resolveMeasurementJob");async function R(t,e,i=()=>{}){let{eval_:r,configuration:o,targetContent:u,runNumber:s}=t;i(`[${o}] ${r.name} run#${s} \u2014 executing...`);let l=x(e.engine),g=w({targetType:e.targetType,targetContent:u,targetName:e.targetName,engine:l}),c={...e.executionKey?{executionKey:e.executionKey}:{},deliveryRegime:`${e.targetType??"prompt"}:${l}:${g.methodology}`,...e.measurement?{measurement:e.measurement}:{},methodology:g.methodology,delivery:g.delivery,isolated:g.isWorktree,...e.candidateContentHash?{candidateContentHash:e.candidateContentHash}:{},...g.leakedFiles.length?{leakedFiles:g.leakedFiles}:{}};g.isWorktree||i(`[${o}] ${r.name} run#${s} \u2014 \u26A0 no git worktree; running without repo context`),g.leakedFiles.length&&i(`[${o}] ${r.name} run#${s} \u2014 \u26A0 CONTAMINATED: ${g.leakedFiles.join(", ")} survived the strip`);try{let n=await b({prompt:r.prompt,targetContent:g.systemPrompt,workDir:g.workDir,timeout:e.timeout,model:e.model,engine:l});if(n.error){let d=C(n.error);return i(`[${o}] ${r.name} run#${s} \u2014 FAILED (${d.kind}): ${n.error}`),{evalId:r.id,evalTargetId:e.targetId,evalTargetVersion:e.targetVersion,configuration:o,runNumber:s,iteration:e.iteration,status:"failed",outputs:{error:n.error,failureKind:d.kind,failureClass:d.class,...c,engine:n.engine??l,model:n.model??null,...n.provider?{provider:n.provider}:{},...n.costUsd!=null?{costUsd:n.costUsd}:{}},timing:{duration_ms:n.durationMs,total_tokens:n.totalTokens},grading:{},failure:d}}i(`[${o}] ${r.name} run#${s} \u2014 completed in ${(n.durationMs/1e3).toFixed(1)}s`);let a={};if(!e.skipGrading&&r.expectations.length>0){i(`[${o}] ${r.name} run#${s} \u2014 grading...`);let d=await M({prompt:r.prompt,output:n.output,expectations:r.expectations,...$({engine:l,model:e.model},{engine:e.graderEngine,model:e.graderModel}),timeout:e.timeout});a=d,i(`[${o}] ${r.name} run#${s} \u2014 grading done: pass_rate=${d.summary.pass_rate==null?"n/a (all assertions errored)":`${(d.summary.pass_rate*100).toFixed(0)}%`}`)}return{evalId:r.id,evalTargetId:e.targetId,evalTargetVersion:e.targetVersion,configuration:o,runNumber:s,iteration:e.iteration,status:"completed",outputs:{transcript:n.output,...c,engine:n.engine??l,...n.provider?{provider:n.provider}:{},model:n.model??null,...n.costUsd!=null?{costUsd:n.costUsd}:{}},timing:{duration_ms:n.durationMs,total_tokens:n.totalTokens,total_duration_seconds:n.durationMs/1e3},grading:a}}finally{g.cleanup()}}m(R,"executeAndGradeCase");async function B(t){let{cases:e,targetContent:i,concurrency:r,runNumber:o,opts:u,skipBaseline:s,onLog:l=m(()=>{},"onLog")}=t,g=s?["with_target"]:["with_target","without_target"],c=[];for(let a of e)for(let d of g)c.push({eval_:a,configuration:d,targetContent:d==="with_target"?i:void 0,runNumber:o});let n=[];for(let a=0;a<c.length;a+=Math.max(1,r)){let d=c.slice(a,a+Math.max(1,r)),h=await Promise.all(d.map(f=>R(f,u,l)));n.push(...h);let p=[];for(let f of h)try{await v("/api/eval-runs",f)}catch(E){p.push(E instanceof Error?E.message:String(E))}if(p.length)throw new Error(`Could not persist ${p.length} eval run(s); stopped further execution: ${p.join("; ")}`)}try{await v(`/api/eval-targets/${u.targetId}/label-state`,{})}catch(a){l(`  \u26A0 label-state notification skipped: ${a instanceof Error?a.message:String(a)}`)}return{completed:n.filter(a=>a.status==="completed").length,failed:n.filter(a=>a.status==="failed").length,results:n}}m(B,"runBatchAndPost");export{$ as a,_ as b,A as c,R as d,B as e};

package/dist/{chunk-DVU5PHPM.js → chunk-FG343JM3.js}

@@ -1,4 +1,4 @@
-import{a as w,b as S}from"./chunk-CDY7TQFZ.js";import{f as A}from"./chunk-FBSZ6KCM.js";import{d as G}from"./chunk-SG4UDDHB.js";import{a as p}from"./chunk-ZLLER3HP.js";var j=`You are a rigorous eval grader. Your job is to evaluate whether an AI assistant's output meets a set of expectations.
+import{a as w,b as S}from"./chunk-CDY7TQFZ.js";import{f as A}from"./chunk-4GOXEEXZ.js";import{d as G}from"./chunk-JNLDISBA.js";import{a as p}from"./chunk-ZLLER3HP.js";var j=`You are a rigorous eval grader. Your job is to evaluate whether an AI assistant's output meets a set of expectations.
 
 You will receive:
 1. The original prompt that was given to the AI

package/dist/chunk-HT6R6KR3.js

@@ -0,0 +1,3 @@
+import{a as s}from"./chunk-ZLLER3HP.js";var _=1.959963984540054;function k(n,t,e=_){if(t===0)return{lower:0,upper:1};let o=n/t,r=e*e,i=1+r/t,u=(o+r/(2*t))/i,a=e/i*Math.sqrt(o*(1-o)/t+r/(4*t*t));return{lower:Math.max(0,u-a),upper:Math.min(1,u+a)}}s(k,"wilsonInterval");function L(n,t){let e=n,o=t,r=e+o;if(r===0)return{pValue:1,nDiscordant:0};let i=Math.min(e,o),u=Math.pow(.5,r),a=u;for(let c=1;c<=i;c++)u*=(r-c+1)/c,a+=u;return{pValue:Math.min(1,2*a),nDiscordant:r}}s(L,"mcnemarExact");function U(n,t,e=10){return{sufficient:n>=e&&t>=e,minN:e,nWith:n,nWithout:t,shortfall:{with:Math.max(0,e-n),without:Math.max(0,e-t)}}}s(U,"assessSufficiency");function M(n){let t=n>>>0;return()=>{t=t+1831565813|0;let e=Math.imul(t^t>>>15,1|t);return e=e+Math.imul(e^e>>>7,61|e)^e,((e^e>>>14)>>>0)/4294967296}}s(M,"seededRandom");function C(n,t){let e=n.length;if(e===1)return n[0];let o=(e-1)*t,r=Math.floor(o),i=Math.ceil(o);return r===i?n[r]:n[r]+(o-r)*(n[i]-n[r])}s(C,"sortedQuantile");function j(n,t={}){let e=n.length;if(e===0)return{estimate:0,lower:0,upper:0,iterations:0};let o=t.iterations??1e4,r=t.ciLevel??.95,i=M(t.seed??1),u=n.reduce((l,f)=>l+f,0)/e,a=new Array(o);for(let l=0;l<o;l++){let f=0;for(let h=0;h<e;h++)f+=n[i()*e|0];a[l]=f/e}a.sort((l,f)=>l-f);let c=(1-r)/2;return{estimate:u,lower:C(a,c),upper:C(a,1-c),iterations:o}}s(j,"pairedBootstrapCI");function V(n,t={}){let e=n.filter(d=>d.withScores.length>0&&d.withoutScores.length>0);if(e.length===0)return{estimate:0,lower:0,upper:0,pValue:1,iterations:0};let o=s(d=>d.reduce((p,g)=>p+g,0)/d.length,"mean"),r=e.map(d=>d.withScores.length*d.withoutScores.length/(d.withScores.length+d.withoutScores.length)),i=r.reduce((d,p)=>d+p,0),u=s(d=>d.reduce((p,g,w)=>p+r[w]*g,0)/i,"combine"),a=u(e.map(d=>o(d.withScores)-o(d.withoutScores))),c=t.iterations??1e4,l=t.ciLevel??.95,f=M(t.seed??1),h=new Array(c),b=0,x=0;for(let d=0;d<c;d++){let p=e.map(w=>{let $=0;for(let v=0;v<w.withScores.length;v++)$+=w.withScores[f()*w.withScores.length|0];let T=0;for(let v=0;v<w.withoutScores.length;v++)T+=w.withoutScores[f()*w.withoutScores.length|0];return $/w.withScores.length-T/w.withoutScores.length}),g=u(p);h[d]=g,g<=0&&b++,g>=0&&x++}h.sort((d,p)=>d-p);let A=(1-l)/2;return{estimate:a,lower:C(h,A),upper:C(h,1-A),pValue:Math.min(1,2*Math.min(b,x)/c),iterations:c}}s(V,"stratifiedBootstrapCI");function Q(n,t){let e=n.length,o=t.length;if(e===0||o===0)return 0;let r=[...n.map(l=>({t:l,exposed:!0})),...t.map(l=>({t:l,exposed:!1}))].sort((l,f)=>l.t-f.t),i=0,u=0;for(;u<r.length;){let l=u;for(;l+1<r.length&&r[l+1].t===r[u].t;)l++;let f=(u+1+l+1)/2;for(let h=u;h<=l;h++)r[h].exposed&&(i+=f);u=l+1}let c=(i-e*(e+1)/2)/(e*o);return Math.abs(c-.5)}s(Q,"cohortTimeSeparation");var N=.35,D=900*1e3;function W(n,t,e=N,o=D){if(n.length===0||t.length===0)return!1;let r=[...n,...t];return Math.max(...r)-Math.min(...r)<o?!1:Q(n,t)>e}s(W,"isTimeSeparated");var K=s(n=>n===0||n===1,"isBinary");function ee(n,t,e){return n.every(r=>r[t]!=null&&r[e]!=null)?n.reduce((r,i)=>r+(i[t]-i[e]),0)/n.length:null}s(ee,"meanPairDelta");function q(n,t={}){let e=t.baseline??"no-rule",o=t.minN??10,r=n.length,i=n.map(d=>d.withScore-d.withoutScore),u=r===0?0:i.reduce((d,p)=>d+p,0)/r,a=j(i,{seed:t.seed,iterations:t.iterations}),c={lower:a.lower,upper:a.upper},l=n.every(d=>K(d.withScore)&&K(d.withoutScore)),f=null;if(l&&r>0){let d=0,p=0;for(let g of n)g.withScore===1&&g.withoutScore===0?d++:g.withScore===0&&g.withoutScore===1&&p++;f=L(d,p).pValue}let{sufficient:h}=U(r,r,o),b=c.lower>0||c.upper<0,A={mode:"controlled",baseline:e,delta:u,ci:c,pValue:f,significant:h&&b,sufficient:h,nWith:r,nWithout:r,costDelta:ee(n,"withCost","withoutCost"),latencyDelta:ee(n,"withLatency","withoutLatency"),confounded:!1};return h||(A.insufficientReason=`insufficient data: need >= ${o} paired tasks per cohort, have ${r}`),A}s(q,"buildControlledVerdict");var we=2;function Y(n,t={}){let e=t.minN??10,o=t.minPerStratum??we,r=t.maxTimeSeparation??N,i=t.minStratumSpanMs??D,u=n.filter(m=>m.withScores.length>=o&&m.withoutScores.length>=o),a=u.filter(m=>m.withTimes!=null&&m.withoutTimes!=null&&W(m.withTimes,m.withoutTimes,r,i)),c=new Set(a.map(m=>m.key)),l=u.filter(m=>!c.has(m.key)),f=n.filter(m=>m.withScores.length>0&&m.withoutScores.length>0&&(m.withScores.length<o||m.withoutScores.length<o)).length,h=l.reduce((m,S)=>m+S.withScores.length,0),b=l.reduce((m,S)=>m+S.withoutScores.length,0),x=V(l,{seed:t.seed,iterations:t.iterations}),A={lower:x.lower,upper:x.upper},d=x.estimate,p=l.length>0&&h>=e&&b>=e,g=l.length>0?x.pValue:null,w=p&&(A.lower>0||A.upper<0),$=n.reduce((m,S)=>m+S.withScores.length+S.withoutScores.length,0),T=$>0?(h+b)/$:0,v={mode:"observational",baseline:t.baseline??"no-rule",delta:d,ci:A,pValue:g,significant:w,sufficient:p,nWith:h,nWithout:b,costDelta:null,latencyDelta:null,confounded:!0,strata:l.length,strataDroppedForTimeSeparation:a.length,coverage:T};if(!p){let m=f>0?` (${f} project(s) had both cohorts but fewer than ${o} sessions in one of them)`:"",S=a.length>0?` ${a.length} project-month(s) held both cohorts but their sessions are separated in time (exposure predicts when a session ran), so the difference within them is a calendar difference, not a treatment effect`:"";v.insufficientReason=l.length===0?a.length>0?`insufficient data: no comparable cohort.${S}${m}`:`insufficient data: no project has >= ${o} exposed AND >= ${o} unexposed sessions to compare within${m}`:`insufficient data: need >= ${e} sessions per cohort across comparable projects, have ${h} exposed / ${b} unexposed${m}${S?`.${S}`:""}`}return v}s(Y,"buildStratifiedVerdict");function te(n){let t=new Map;for(let e of n){let o=t.get(e.stratum);o||(o={key:e.stratum,withScores:[],withoutScores:[],withTimes:[],withoutTimes:[]},t.set(e.stratum,o)),(e.exposed?o.withScores:o.withoutScores).push(e.score),e.time!=null&&(e.exposed?o.withTimes:o.withoutTimes).push(e.time)}for(let e of t.values())(e.withTimes.length!==e.withScores.length||e.withoutTimes.length!==e.withoutScores.length)&&(delete e.withTimes,delete e.withoutTimes);return[...t.values()]}s(te,"toStrata");var R=.05;function ne(n,t={}){let e=t.trialIndex??0,o=t.seed??1,r=t.exposedFraction??.5,i=t.effect??0,u=M(o+e*2),a=n.map(c=>{let l=u()<r;return{stratum:c.stratum,score:l?c.score+i:c.score,exposed:l,time:c.time}});return Y(te(a),{seed:o+e*2+1,iterations:t.iterations??500,minN:t.minN,minPerStratum:t.minPerStratum})}s(ne,"runCalibrationTrial");function B(n,t={}){let e=t.trials??200,o=[];for(let r=0;r<e;r++)o.push(ne(n,{...t,trialIndex:r}));return I(o,t.effect??0)}s(B,"calibrate");function I(n,t){let e=0,o=0,r=0,i=0;for(let c of n)c.sufficient&&(e++,c.significant&&o++,r+=c.delta,i=Math.max(i,Math.abs(c.delta)));let u=e>0?o/e:0,a=k(o,e,_);return{trials:n.length,evaluated:e,insufficient:n.length-e,significantTrials:o,significantRate:u,significantRateCi:a,meanDelta:e>0?r/e:0,maxAbsDelta:i,alpha:R,effect:t,calibrated:t===0&&e>0&&a.lower<=R&&R<=a.upper}}s(I,"summarizeTrials");function P(n,t={}){return B(n,{...t,effect:0})}s(P,"aaCalibration");function Z(n,t,e={}){return B(n,{...e,effect:t})}s(Z,"powerCheck");var be=.8416212335729143;var re=[{ym:"2026-03",n:122,mean:.287},{ym:"2026-04",n:50,mean:.07},{ym:"2026-05",n:267,mean:.089},{ym:"2026-06",n:61,mean:.201},{ym:"2026-07",n:162,mean:.071}],X=662;function H(n=2){let t=[];for(let e of re){let o=Math.round(e.n*e.mean);for(let r=0;r<e.n;r++)t.push({stratum:`proj-${r%n}|${e.ym}`,score:r<o?1:0})}if(n===2&&t.length!==X)throw new Error(`realisticCanaryCorpus: expected ${X} sessions, built ${t.length}`);return t}s(H,"realisticCanaryCorpus");function ie(n=600){return Array.from({length:n},(t,e)=>({stratum:"only",score:e%4===0?1:0}))}s(ie,"flatCanaryCorpus");var ae=50,se=4,F={min:.2,max:.7};function Se(n,t){let e=M(t),o=F.max-F.min;return Array.from({length:n},()=>F.min+e()*o)}s(Se,"caseDifficulties");function oe(n,t,e){let o=0;for(let r=0;r<t;r++)e()<n&&o++;return o/t}s(oe,"passRate");function ce(n={}){let t=n.trialIndex??0,e=n.seed??1,o=n.cases??ae,r=n.assertions??se,i=n.effect??0,u=Se(o,e),a=M(e+t*2),c=u.map(l=>({withScore:oe(l+i,r,a),withoutScore:oe(l,r,a)}));return q(c,{baseline:"prior-version",seed:e+t*2+1,iterations:n.iterations??500,minN:n.minN})}s(ce,"runControlledCalibrationTrial");function G(n={}){let t=n.trials??200,e=[];for(let o=0;o<t;o++)e.push(ce({...n,trialIndex:o}));return I(e,n.effect??0)}s(G,"controlledCalibration");function le(n={}){return G({...n,effect:0})}s(le,"controlledAaCalibration");function ue(n,t={}){return G({...t,effect:n})}s(ue,"controlledPowerCheck");var de={maxSignificantRate:.15,maxMeanAbsDelta:.02,maxWorstCaseDelta:.15,powerProbeEffect:.2,minPower:.9,minPowerMeanDelta:.15};function E(n,t,e){let o=e.filter(r=>!r.ok).map(r=>r.why);return{name:n,passed:o.length===0,detail:o.length===0?`${n}: ok (rate=${t.significantRate.toFixed(3)}, meanDelta=${t.meanDelta.toFixed(4)}, evaluated=${t.evaluated}/${t.trials})`:`${n}: ${o.join("; ")}`,result:t}}s(E,"check");function Ae(n={}){let t=n.trials??200,e=n.powerTrials??200,o=de,r=[],i=P(ie(),{trials:t,seed:11});r.push(E("aa-flat",i,[{ok:i.calibrated,why:`\u03B1=${R} outside rate CI [${i.significantRateCi.lower.toFixed(3)}, ${i.significantRateCi.upper.toFixed(3)}]`},{ok:i.significantRate<o.maxSignificantRate,why:`false-positive rate ${i.significantRate.toFixed(3)} \u2265 ${o.maxSignificantRate}`},{ok:Math.abs(i.meanDelta)<o.maxMeanAbsDelta,why:`mean delta ${i.meanDelta.toFixed(4)} not centred on zero`}]));let u=P(H(),{trials:t,seed:23});r.push(E("aa-realistic",u,[{ok:u.calibrated,why:`\u03B1=${R} outside rate CI [${u.significantRateCi.lower.toFixed(3)}, ${u.significantRateCi.upper.toFixed(3)}]`},{ok:u.significantRate<o.maxSignificantRate,why:`false-positive rate ${u.significantRate.toFixed(3)} \u2265 ${o.maxSignificantRate}`},{ok:Math.abs(u.meanDelta)<o.maxMeanAbsDelta,why:`mean delta ${u.meanDelta.toFixed(4)} not centred on zero`},{ok:u.maxAbsDelta<o.maxWorstCaseDelta,why:`worst single-trial delta ${u.maxAbsDelta.toFixed(3)} \u2265 ${o.maxWorstCaseDelta}`}]));let a=Z(H(),o.powerProbeEffect,{trials:e,seed:5});r.push(E("power-observational",a,[{ok:a.significantRate>o.minPower,why:`detects +${o.powerProbeEffect} only ${a.significantRate.toFixed(3)} of the time`},{ok:a.meanDelta>o.minPowerMeanDelta,why:`measured mean delta ${a.meanDelta.toFixed(3)} understates the injected +${o.powerProbeEffect}`}]));let c=le({trials:t,seed:31});r.push(E("aa-controlled",c,[{ok:c.calibrated,why:`\u03B1=${R} outside rate CI [${c.significantRateCi.lower.toFixed(3)}, ${c.significantRateCi.upper.toFixed(3)}]`},{ok:c.significantRate<o.maxSignificantRate,why:`false-positive rate ${c.significantRate.toFixed(3)} \u2265 ${o.maxSignificantRate}`},{ok:Math.abs(c.meanDelta)<o.maxMeanAbsDelta,why:`mean delta ${c.meanDelta.toFixed(4)} not centred on zero`}]));let l=ue(o.powerProbeEffect,{trials:e,seed:37});r.push(E("power-controlled",l,[{ok:l.significantRate>o.minPower,why:`detects +${o.powerProbeEffect} only ${l.significantRate.toFixed(3)} of the time`}]));let f=r.filter(h=>!h.passed).map(h=>h.detail);return{green:f.length===0,checks:r,failures:f}}s(Ae,"runPromotionCanary");var Me=3,Re=7,J="(preamble)",_e=/^\s*(`{3,}|~{3,})/,$e=/^(#{1,6})\s+(.*)$/;function y(n){let t=n.split(`
+`),e=[],o=[],r={heading:"",level:0,path:[J],body:[]},i=null,u=s(()=>{e.push({heading:r.heading,level:r.level,path:r.path,body:r.body.join(`
+`).trim()})},"flush");for(let a of t){let c=_e.exec(a);if(c){let f=c[1];i==null?i={marker:f}:f[0]===i.marker[0]&&f.length>=i.marker.length&&(i=null),r.body.push(a);continue}if(i!=null){r.body.push(a);continue}let l=$e.exec(a);if(l){u();let f=l[1].length,h=l[2].trim();o.length=Math.max(0,f-1),o[f-1]=h,r={heading:h,level:f,path:Array.from({length:f},(b,x)=>o[x]??`(unnamed level ${x+1})`),body:[]};continue}r.body.push(a)}return u(),e.filter((a,c)=>!(c===0&&a.level===0&&a.body===""))}s(y,"splitMarkdownSections");function O(n){let t=new Map,e=new Map;for(let o of n){let r=o.level===0?J:o.path.join(" > "),i=(t.get(r)??0)+1;t.set(r,i),e.set(i===1?r:`${r} #${i}`,o)}return e}s(O,"sectionKeys");function he(n,t){let e=O(y(n)),o=O(y(t)),r=[],i=[],u=[];for(let[a,c]of o){let l=e.get(a);l?l.body!==c.body&&r.push(a):i.push(a)}for(let a of e.keys())o.has(a)||u.push(a);return{changed:r.sort(),added:i.sort(),removed:u.sort()}}s(he,"diffSections");function me(n){return[...new Set([...n.changed,...n.added,...n.removed])].sort()}s(me,"touchedSections");var fe=s(n=>n.trim().toLowerCase(),"normalize");function Ee(n,t,e){if(e.length===0)return[];let o=new Set(e.map(fe)),r=he(n,t),i=new Set(me(r)),u=new Map([...O(y(n)),...O(y(t))]),a=[];for(let c of i){let l=u.get(c);(l?l.path:c.split(" > ")).some(h=>o.has(fe(h)))&&a.push(c)}return a.sort()}s(Ee,"protectedViolations");export{_ as a,q as b,be as c,Ae as d,Me as e,Re as f,Ee as g};

package/dist/{chunk-DXNUCWUX.js → chunk-HXOQUYG3.js}

@@ -1,4 +1,4 @@
-import{b as p}from"./chunk-ZK3ACE4T.js";import{g as l}from"./chunk-7BCZDVIU.js";import{a}from"./chunk-ZLLER3HP.js";var g=5,h=a(e=>e==null?"n/a":`${Math.round(e*100)}%`,"pct");function b(e){return!e||e.length===0?"":`
+import{b as p}from"./chunk-T7HZZ4DJ.js";import{g as l}from"./chunk-XCH4HPEZ.js";import{a}from"./chunk-ZLLER3HP.js";var g=5,h=a(e=>e==null?"n/a":`${Math.round(e*100)}%`,"pct");function b(e){return!e||e.length===0?"":`
 
 ## Prior cases that FAILED to discriminate \u2014 do NOT reproduce these
 The cases below were run WITH and WITHOUT the instruction and did NOT separate the two

package/dist/{chunk-HM63AB2C.js → chunk-IFIWDL7G.js}

@@ -1,4 +1,4 @@
-import{h as o}from"./chunk-CVPAKEEZ.js";import{g as f}from"./chunk-7BCZDVIU.js";import{b as y,e as m,h as d}from"./chunk-GIXHG5FK.js";import{a as S}from"./chunk-ZLLER3HP.js";var _=1800*1e3,G=300*1e3,R=10,T=`Given these messages from an AI coding session, write a 1-2 sentence summary of what the user is trying to accomplish. Focus on the goal, not the steps. No preamble.
+import{h as o}from"./chunk-CVPAKEEZ.js";import{g as f}from"./chunk-XCH4HPEZ.js";import{b as y,e as m,h as d}from"./chunk-GIXHG5FK.js";import{a as S}from"./chunk-ZLLER3HP.js";var _=1800*1e3,G=300*1e3,R=10,T=`Given these messages from an AI coding session, write a 1-2 sentence summary of what the user is trying to accomplish. Focus on the goal, not the steps. No preamble.
 
 Return strict JSON in the form: {"goalSummary": "..."}`;async function N(n,e={}){let w=e.llm??f,i=e.now??Date.now,v=e.getSession??(async s=>{try{return await o(()=>m(`/api/sessions/${s}`),{label:"session detail",idempotent:!0})}catch(t){if(t instanceof y&&t.statusCode===404)return null;throw t}}),l=e.getEvents??((s,t)=>o(()=>m(`/api/sessions/${s}/events?limit=${t.limit}&sort=${t.sort}`),{label:"session events",idempotent:!0})),P=e.saveGoal??(async(s,t)=>{await o(()=>d(`/api/sessions/${s}/enrich`,{analysis:t}),{label:"session enrich"})}),r=await v(n.sessionId);if(!r)return{skipped:"not_found"};let[A]=await l(n.sessionId,{limit:1,sort:"desc"}),h=new Date(A?.timestamp??r.startedAt).getTime();if(i()-h>_)return{skipped:"stale"};let c=r.analysis??{},g=c.goalSummaryAt?new Date(c.goalSummaryAt).getTime():0;if(g&&i()-g<G)return{skipped:"recent"};let u=(await l(n.sessionId,{limit:R*2,sort:"asc"})).filter(s=>s.content&&(s.eventType==="user_message"||s.eventType==="assistant_response")).map(s=>`${s.eventType==="user_message"?"USER":"ASSISTANT"}: ${s.content}`).join(`
 

package/dist/chunk-JNLDISBA.js

@@ -0,0 +1,4 @@
+import{a as y}from"./chunk-6XTLP2G7.js";import{a as s}from"./chunk-ZLLER3HP.js";import{spawn as v}from"node:child_process";import{mkdtempSync as O,readFileSync as D,rmSync as I}from"node:fs";import{tmpdir as M}from"node:os";import{join as E}from"node:path";var R=s(t=>`codex:${t.replace(/^codex:/,"")}`,"codexModelId");function $(t,o,i=!1){return["exec","--json","--sandbox",i?"workspace-write":"read-only","--skip-git-repo-check","--ephemeral","--model",t.replace(/^codex:/,""),"--output-last-message",o,"-"]}s($,"buildCodexArgs");function A(t){let o;for(let i of t.split(`
+`)){let c=i.trim();if(!c.startsWith("{"))continue;let n;try{n=JSON.parse(c)}catch{continue}if(typeof n!="object"||n===null)continue;let d=n.usage;if(typeof d!="object"||d===null)continue;let u=s(x=>{let p=d[x];return typeof p=="number"&&Number.isFinite(p)?p:0},"num"),l=u("input_tokens"),a=u("output_tokens");o={inputTokens:l,cachedInputTokens:u("cached_input_tokens"),outputTokens:a,reasoningOutputTokens:u("reasoning_output_tokens"),totalTokens:l+a}}return o}s(A,"parseCodexUsage");async function L(t){let{prompt:o,systemPrompt:i,model:c,timeout:n=12e4,workDir:d=process.cwd()}=t,u=i?`${i}
+
+${o}`:o,l=y("codex");if(!l)return{output:"",exitCode:null,durationMs:0,error:"Codex CLI not found on PATH, well-known install dirs, or the node bin dir \u2014 is it installed?"};let a=O(E(M(),"rulemetric-codex-")),x=E(a,"last-message.txt"),p=Date.now();return new Promise(S=>{let f={...process.env};delete f.HTTPS_PROXY;let _=["/opt/homebrew/bin","/usr/local/bin"],T=(f.PATH??"").split(":").filter(Boolean),b=_.filter(e=>!T.includes(e));b.length&&(f.PATH=[...T,...b].join(":"));let r=v(l,$(c,x,t.writableFixture),{cwd:d,env:f,stdio:["pipe","pipe","pipe"]}),C="",m="",w=256*1024;r.stdout.on("data",e=>{m+=e.toString(),m.length>w&&(m=m.slice(-w))}),r.stderr.on("data",e=>{C+=e.toString()});let g=s(e=>{I(a,{recursive:!0,force:!0}),S({...e,durationMs:Date.now()-p})},"finish"),h=setTimeout(()=>{r.kill("SIGTERM"),g({output:"",exitCode:null,error:`codex exec timed out after ${n}ms`})},n);r.on("error",e=>{clearTimeout(h),g({output:"",exitCode:null,error:`failed to spawn codex: ${e.message} (is the Codex CLI installed?)`})}),r.on("close",e=>{clearTimeout(h);let k="";try{k=D(x,"utf8").trim()}catch{}let P=A(m);if(e!==0||k===""){g({output:k,exitCode:e,usage:P,error:`codex exec failed (exit ${e}): ${C.trim().slice(-500)||"no last message produced"}`});return}g({output:k,exitCode:e,model:R(c),usage:P})}),r.stdin.write(u),r.stdin.end()})}s(L,"executeCodexEval");export{R as a,$ as b,A as c,L as d};

package/dist/{chunk-FS3TAQAE.js → chunk-JOXTWLLK.js}

@@ -1,4 +1,4 @@
-import{b as $}from"./chunk-DXNUCWUX.js";import{a as C}from"./chunk-KOYHJFB3.js";import{a as y}from"./chunk-DVU5PHPM.js";import{f}from"./chunk-FBSZ6KCM.js";import{e as h,f as w}from"./chunk-GIXHG5FK.js";import{a as v}from"./chunk-ZLLER3HP.js";async function N(n,i,p){let{conversationId:a,messageId:g,messageContent:r}=n,o=await h(`/api/eval-targets/${i}`),c=await h(`/api/eval-targets/${i}/evals`),e=await h(`/api/eval-conversations/${a}/messages`),t=e.findIndex(l=>l.id===g);if(t>=0&&e.slice(t+1).some(u=>u.role==="assistant"&&(u.type==="results"||u.type==="proposal"||u.type==="generated"||u.type==="text"))){p(`Message ${g} already processed \u2014 skipping (idempotency guard)`);return}let s=v(async(l,u,m)=>{await w(`/api/eval-conversations/${a}/messages/assistant`,{content:l,type:u,data:m})},"postMessage"),d=await _(o,c,e,r,p);p(`Action: ${d.action}`);try{switch(d.action){case"run_evals":await x(o,c,a,s,p);break;case"generate_cases":await k(o,i,a,s,p);break;case"analyze_results":case"propose_changes":await E(o,c,e,r,a,s,p);break;case"respond":await s(d.response,"text");break}}catch(l){let u=l instanceof Error?l.message:String(l);throw await s(`Something went wrong: ${u}`,"text").catch(()=>{}),l}}v(N,"processConversationJob");async function _(n,i,p,a,g){let r=p.slice(-10).map(e=>`${e.role}: ${e.content.slice(0,200)}`).join(`
+import{b as $}from"./chunk-HXOQUYG3.js";import{a as C}from"./chunk-PDXHRF4E.js";import{a as y}from"./chunk-FG343JM3.js";import{f}from"./chunk-4GOXEEXZ.js";import{e as h,f as w}from"./chunk-GIXHG5FK.js";import{a as v}from"./chunk-ZLLER3HP.js";async function N(n,i,p){let{conversationId:a,messageId:g,messageContent:r}=n,o=await h(`/api/eval-targets/${i}`),c=await h(`/api/eval-targets/${i}/evals`),e=await h(`/api/eval-conversations/${a}/messages`),t=e.findIndex(l=>l.id===g);if(t>=0&&e.slice(t+1).some(u=>u.role==="assistant"&&(u.type==="results"||u.type==="proposal"||u.type==="generated"||u.type==="text"))){p(`Message ${g} already processed \u2014 skipping (idempotency guard)`);return}let s=v(async(l,u,m)=>{await w(`/api/eval-conversations/${a}/messages/assistant`,{content:l,type:u,data:m})},"postMessage"),d=await _(o,c,e,r,p);p(`Action: ${d.action}`);try{switch(d.action){case"run_evals":await x(o,c,a,s,p);break;case"generate_cases":await k(o,i,a,s,p);break;case"analyze_results":case"propose_changes":await E(o,c,e,r,a,s,p);break;case"respond":await s(d.response,"text");break}}catch(l){let u=l instanceof Error?l.message:String(l);throw await s(`Something went wrong: ${u}`,"text").catch(()=>{}),l}}v(N,"processConversationJob");async function _(n,i,p,a,g){let r=p.slice(-10).map(e=>`${e.role}: ${e.content.slice(0,200)}`).join(`
 `),o=`You are an eval assistant. Decide what action to take based on the user's message.
 
 ## Current State

package/dist/{chunk-5VNEHPGC.js → chunk-JSGXFD42.js}

@@ -1 +1 @@
-import{a as G,c as I}from"./chunk-T47SFA5X.js";import{f as j}from"./chunk-SWSOEYC7.js";import{a as R}from"./chunk-DVU5PHPM.js";import{b as U}from"./chunk-VBXL7XO7.js";import{f as A}from"./chunk-FBSZ6KCM.js";import{b as P,e as C,h as _}from"./chunk-GIXHG5FK.js";import{a as x}from"./chunk-ZLLER3HP.js";var J=x(l=>l.summary.pass_rate===1?1:0,"fullPassScore");async function z(l,d={}){let $=d.executeEval??A,M=d.grade??R,q=d.getExperiment??(async o=>{try{return await C(`/api/memory-experiments/${o}`)}catch(e){if(e instanceof P&&e.statusCode===404)return null;throw e}}),B=d.getMemoryBody??(async o=>{try{let e=await C(`/api/memories/${o}`);return t?.memoryVersion!=null&&e.version!==t.memoryVersion?null:e.body}catch(e){if(e instanceof P&&e.statusCode===404)return null;throw e}}),p=d.updateExperiment??(async(o,e)=>{await _(`/api/memory-experiments/${o}`,e)}),t=await q(l.experimentId);if(!t)return{skipped:"not_found"};if(t.status==="completed"||t.status==="failed")return{skipped:"already_terminal",status:t.status};let E=t.memoryBody??await B(t.memoryId);if(E==null)return await p(t.id,{status:"failed",error:"memory no longer exists or its queued version is no longer available"}),{skipped:"memory_gone",status:"failed"};await p(t.id,{status:"running"});let v=t.expectations??[],c=t.model??void 0,i=t.engine??"claude",g=G({engine:i,model:c},{engine:l.graderEngine,model:l.graderModel}),f=x(async o=>{let e=(d.createWorkspace??j)({engine:i,targetType:"prompt",targetContent:o});try{if(e.leakedFiles.length)throw new Error("Contaminated memory trial workspace");return await $({prompt:t.taskPrompt,targetContent:e.systemPrompt,appendTarget:!0,ignoreContextFiles:!0,workDir:e.workDir,model:c,engine:i})}finally{e.cleanup()}},"executeArm"),s=[],u=new Set,b,k,m=[],w=0;try{if(l.measurement){let e=I(l);if(e.engine!==i||e.model!==c)throw new Error("Memory experiment differs from its queued measurement snapshot");g={engine:e.graderEngine,model:e.graderModel}}for(let e=0;e<t.trials;e++){let[r,n]=e%2===0?[await f(E),await f()]:await(async()=>{let y=await f();return[await f(E),y]})();if(r.error||n.error||r.exitCode!==0||n.exitCode!==0||!r.output.trim()||!n.output.trim()){w++,m.push({trial:e+1,reason:(r.error??n.error??"Nonzero exit or empty output").slice(0,500)});continue}if(r.engine!==i||n.engine!==i||!r.model||!n.model||r.model!==n.model||r.provider!==n.provider||r.model&&u.size>0&&!u.has(r.model)||i==="pi"&&c&&r.model!==`pi:${c}`){w++,m.push({trial:e+1,reason:"Harness, provider or model changed within the experiment"});continue}r.model&&u.add(r.model),n.model&&u.add(n.model);let[a,h]=await Promise.all([M({prompt:t.taskPrompt,output:r.output,expectations:v,...g}),M({prompt:t.taskPrompt,output:n.output,expectations:v,...g})]),T=JSON.stringify([a.judge_engine??null,a.judge_model??null]);if([a,h].some(y=>y.summary.pass_rate==null||y.summary.errored||y.summary.total!==v.length)||a.judge_model!==h.judge_model||a.judge_engine!==h.judge_engine||b!==void 0&&b!==T){w++,m.push({trial:e+1,reason:"Grading incomplete or judge changed; not a model failure"});continue}b=T,k={engine:a.judge_engine??null,model:a.judge_model??null},s.push({withScore:J(a),withoutScore:J(h),withLatency:r.durationMs/1e3,withoutLatency:n.durationMs/1e3,withTokens:r.totalTokens,withoutTokens:n.totalTokens,...r.costUsd!==void 0?{withCostUsd:r.costUsd}:{},...n.costUsd!==void 0?{withoutCostUsd:n.costUsd}:{}})}if(s.length===0){let e=`no trials completed (${w}/${t.trials} runs errored \u2014 is the ${i} CLI available?)`;return await p(t.id,{status:"failed",error:e,result:{pairs:s,excluded:m,completedTrials:0,engine:i}}),{status:"failed",requestedTrials:t.trials,completedTrials:0}}let o=U(s.map(e=>({...e,withCost:e.withCostUsd,withoutCost:e.withoutCostUsd})));return await p(t.id,{status:"completed",result:{verdict:o,pairs:s,excluded:m,judge:g,actualJudge:k,methodology:"v2-neutral-appended-memory",completedTrials:s.length,engine:i,model:[...u][0]}}),{status:"completed",requestedTrials:t.trials,completedTrials:s.length,significant:o.significant,delta:o.delta}}catch(o){let e=o instanceof Error?o.message:String(o);return await p(t.id,{status:"failed",error:e.slice(0,500),result:{pairs:s,excluded:m,completedTrials:s.length,engine:i,model:[...u][0]??null,judge:g,actualJudge:k,methodology:"v2-neutral-appended-memory"}}).catch(()=>{}),{status:"failed",requestedTrials:t.trials,completedTrials:s.length}}}x(z,"processMemoryAb");export{z as a};
+import{a as G,c as I}from"./chunk-FADINV4H.js";import{f as j}from"./chunk-SWSOEYC7.js";import{a as R}from"./chunk-FG343JM3.js";import{b as U}from"./chunk-HT6R6KR3.js";import{f as A}from"./chunk-4GOXEEXZ.js";import{b as P,e as C,h as _}from"./chunk-GIXHG5FK.js";import{a as x}from"./chunk-ZLLER3HP.js";var J=x(l=>l.summary.pass_rate===1?1:0,"fullPassScore");async function z(l,d={}){let $=d.executeEval??A,M=d.grade??R,q=d.getExperiment??(async o=>{try{return await C(`/api/memory-experiments/${o}`)}catch(e){if(e instanceof P&&e.statusCode===404)return null;throw e}}),B=d.getMemoryBody??(async o=>{try{let e=await C(`/api/memories/${o}`);return t?.memoryVersion!=null&&e.version!==t.memoryVersion?null:e.body}catch(e){if(e instanceof P&&e.statusCode===404)return null;throw e}}),p=d.updateExperiment??(async(o,e)=>{await _(`/api/memory-experiments/${o}`,e)}),t=await q(l.experimentId);if(!t)return{skipped:"not_found"};if(t.status==="completed"||t.status==="failed")return{skipped:"already_terminal",status:t.status};let E=t.memoryBody??await B(t.memoryId);if(E==null)return await p(t.id,{status:"failed",error:"memory no longer exists or its queued version is no longer available"}),{skipped:"memory_gone",status:"failed"};await p(t.id,{status:"running"});let v=t.expectations??[],c=t.model??void 0,i=t.engine??"claude",g=G({engine:i,model:c},{engine:l.graderEngine,model:l.graderModel}),f=x(async o=>{let e=(d.createWorkspace??j)({engine:i,targetType:"prompt",targetContent:o});try{if(e.leakedFiles.length)throw new Error("Contaminated memory trial workspace");return await $({prompt:t.taskPrompt,targetContent:e.systemPrompt,appendTarget:!0,ignoreContextFiles:!0,workDir:e.workDir,model:c,engine:i})}finally{e.cleanup()}},"executeArm"),s=[],u=new Set,b,k,m=[],w=0;try{if(l.measurement){let e=I(l);if(e.engine!==i||e.model!==c)throw new Error("Memory experiment differs from its queued measurement snapshot");g={engine:e.graderEngine,model:e.graderModel}}for(let e=0;e<t.trials;e++){let[r,n]=e%2===0?[await f(E),await f()]:await(async()=>{let y=await f();return[await f(E),y]})();if(r.error||n.error||r.exitCode!==0||n.exitCode!==0||!r.output.trim()||!n.output.trim()){w++,m.push({trial:e+1,reason:(r.error??n.error??"Nonzero exit or empty output").slice(0,500)});continue}if(r.engine!==i||n.engine!==i||!r.model||!n.model||r.model!==n.model||r.provider!==n.provider||r.model&&u.size>0&&!u.has(r.model)||i==="pi"&&c&&r.model!==`pi:${c}`){w++,m.push({trial:e+1,reason:"Harness, provider or model changed within the experiment"});continue}r.model&&u.add(r.model),n.model&&u.add(n.model);let[a,h]=await Promise.all([M({prompt:t.taskPrompt,output:r.output,expectations:v,...g}),M({prompt:t.taskPrompt,output:n.output,expectations:v,...g})]),T=JSON.stringify([a.judge_engine??null,a.judge_model??null]);if([a,h].some(y=>y.summary.pass_rate==null||y.summary.errored||y.summary.total!==v.length)||a.judge_model!==h.judge_model||a.judge_engine!==h.judge_engine||b!==void 0&&b!==T){w++,m.push({trial:e+1,reason:"Grading incomplete or judge changed; not a model failure"});continue}b=T,k={engine:a.judge_engine??null,model:a.judge_model??null},s.push({withScore:J(a),withoutScore:J(h),withLatency:r.durationMs/1e3,withoutLatency:n.durationMs/1e3,withTokens:r.totalTokens,withoutTokens:n.totalTokens,...r.costUsd!==void 0?{withCostUsd:r.costUsd}:{},...n.costUsd!==void 0?{withoutCostUsd:n.costUsd}:{}})}if(s.length===0){let e=`no trials completed (${w}/${t.trials} runs errored \u2014 is the ${i} CLI available?)`;return await p(t.id,{status:"failed",error:e,result:{pairs:s,excluded:m,completedTrials:0,engine:i}}),{status:"failed",requestedTrials:t.trials,completedTrials:0}}let o=U(s.map(e=>({...e,withCost:e.withCostUsd,withoutCost:e.withoutCostUsd})));return await p(t.id,{status:"completed",result:{verdict:o,pairs:s,excluded:m,judge:g,actualJudge:k,methodology:"v2-neutral-appended-memory",completedTrials:s.length,engine:i,model:[...u][0]}}),{status:"completed",requestedTrials:t.trials,completedTrials:s.length,significant:o.significant,delta:o.delta}}catch(o){let e=o instanceof Error?o.message:String(o);return await p(t.id,{status:"failed",error:e.slice(0,500),result:{pairs:s,excluded:m,completedTrials:s.length,engine:i,model:[...u][0]??null,judge:g,actualJudge:k,methodology:"v2-neutral-appended-memory"}}).catch(()=>{}),{status:"failed",requestedTrials:t.trials,completedTrials:s.length}}}x(z,"processMemoryAb");export{z as a};

package/dist/chunk-JU5ZNXPK.js

@@ -0,0 +1 @@
+import{a as i}from"./chunk-ZLLER3HP.js";import{mkdirSync as y,writeFileSync as p,readFileSync as u,existsSync as a,lstatSync as m,statSync as g}from"node:fs";import{dirname as h,join as x}from"node:path";import{createHash as S}from"node:crypto";var l=i(t=>S("sha256").update(JSON.stringify(t)).digest("hex"),"fixtureHash");function c(t,e){if(e.startsWith("/")||e.includes("\\")||e.split("/").some(r=>!r||r===".."||r==="."))throw new Error("Unsafe fixture path");let n=t;for(let r of e.split("/")){n=x(n,r);try{if(m(n).isSymbolicLink())throw new Error("Fixture path is a symlink")}catch(o){if(o.code!=="ENOENT")throw o}}return n}i(c,"safePath");function E(t,e){for(let n of e){let r=c(t,n.path);y(h(r),{recursive:!0}),p(r,n.content)}}i(E,"installFixture");function O(t,e){let n=e.checks.map(r=>{let o=null,s=!1;try{let f=c(t,r.path);r.kind==="file_absent"?s=!a(f):a(f)&&g(f).size<=2e5&&(o=u(f,"utf8"),s=r.kind==="text_equals"?o===r.expected:d(JSON.parse(o),JSON.parse(r.expected)))}catch{s=!1}return{name:r.name,passed:s,actual:o}});return{checks:n,primaryPassed:n.every((r,o)=>!e.checks[o].primary||r.passed),safeguardsPassed:n.every((r,o)=>e.checks[o].primary||r.passed)}}i(O,"checkFixture");function d(t,e){if(t===e)return!0;if(!t||!e||typeof t!="object"||typeof e!="object"||Array.isArray(t)!==Array.isArray(e))return!1;let n=Object.keys(t),r=Object.keys(e);return n.length===r.length&&n.every(o=>Object.prototype.hasOwnProperty.call(e,o)&&d(t[o],e[o]))}i(d,"equalJson");function b(t,e){return e.error?e.error:e.engine!==t.scope.engine?"Observed engine differs from protocol":e.engine==="codex"?"Codex model provenance is requested-only; confirmatory inference refused":e.model!==t.scope.model?"Observed model differs from protocol or is unknown":null}i(b,"studyExclusion");function A(t,e,n,r){for(let f of[...e.files,...n.files])if(u(c(t,f.path),"utf8")!==f.content)throw new Error("Installed fixture differs from frozen bytes");let o=c(t,r==="claude"?"CLAUDE.md":"AGENTS.md"),s=a(o)?u(o,"utf8"):"";if(s!==n.content)throw new Error("Delivered instruction differs from frozen bytes");return{deliveryHash:l({...n,content:s}),fixtureHash:l(e.files)}}i(A,"verifyFixtureDelivery");export{l as a,E as b,O as c,b as d,A as e};

package/dist/{chunk-H4QPAEEA.js → chunk-JUBXWTJA.js}

@@ -1 +1 @@
-import{b as u,c as d}from"./chunk-RR7V5CK5.js";import{b as g}from"./chunk-DXNUCWUX.js";import{e as m,f as p,h as l}from"./chunk-GIXHG5FK.js";import{a as s}from"./chunk-ZLLER3HP.js";async function x(f,R){let r=s(e=>R?.logger?.info?.(e)??console.log(e),"log"),{evalTargetId:n}=f,i=`[cron_refine_cases] ${n}`,t=await m(`/api/eval-targets/${n}`);if(!(t.metadata?.autoRefineCases===!0))return r(`${i} not opted in (metadata.autoRefineCases)`),{skipped:"not-opted-in",reason:`"${t.name}" has not opted into automatic case refinement. Set metadata.autoRefineCases=true to let the loop cull cases that carry no treatment signal, or run \`rulemetric evals refine-cases ${n}\` by hand.`,targetName:t.name};let C=await m(`/api/eval-targets/${n}/judge-calibration`),a=u(C.caseDiscrimination);if(a.refused)return r(`${i} ${a.refused}: ${a.summary}`),{skipped:a.refused,reason:a.summary,targetName:t.name};r(`${i} ${a.summary}`);let o=await d(a,{generate:s(e=>g({content:t.content,name:t.name,feedback:e}),"generate"),createCase:s(e=>p(`/api/eval-targets/${n}/evals`,{name:e.name,prompt:e.prompt,expectations:e.expectations}),"createCase"),retireCase:s(async(e,y)=>{await l(`/api/eval-targets/${n}/evals/${e}`,{retire:!0,retiredReason:y})},"retireCase"),log:r});r(`${i} ${o.summary}`);let{skipped:c,...$}=o;return{...$,targetName:t.name,...c?{skipped:c,reason:o.summary}:{}}}s(x,"cronRefineCases");export{x as a};
+import{b as u,c as d}from"./chunk-EJY5GK6A.js";import{b as g}from"./chunk-HXOQUYG3.js";import{e as m,f as p,h as l}from"./chunk-GIXHG5FK.js";import{a as s}from"./chunk-ZLLER3HP.js";async function x(f,R){let r=s(e=>R?.logger?.info?.(e)??console.log(e),"log"),{evalTargetId:n}=f,i=`[cron_refine_cases] ${n}`,t=await m(`/api/eval-targets/${n}`);if(!(t.metadata?.autoRefineCases===!0))return r(`${i} not opted in (metadata.autoRefineCases)`),{skipped:"not-opted-in",reason:`"${t.name}" has not opted into automatic case refinement. Set metadata.autoRefineCases=true to let the loop cull cases that carry no treatment signal, or run \`rulemetric evals refine-cases ${n}\` by hand.`,targetName:t.name};let C=await m(`/api/eval-targets/${n}/judge-calibration`),a=u(C.caseDiscrimination);if(a.refused)return r(`${i} ${a.refused}: ${a.summary}`),{skipped:a.refused,reason:a.summary,targetName:t.name};r(`${i} ${a.summary}`);let o=await d(a,{generate:s(e=>g({content:t.content,name:t.name,feedback:e}),"generate"),createCase:s(e=>p(`/api/eval-targets/${n}/evals`,{name:e.name,prompt:e.prompt,expectations:e.expectations}),"createCase"),retireCase:s(async(e,y)=>{await l(`/api/eval-targets/${n}/evals/${e}`,{retire:!0,retiredReason:y})},"retireCase"),log:r});r(`${i} ${o.summary}`);let{skipped:c,...$}=o;return{...$,targetName:t.name,...c?{skipped:c,reason:o.summary}:{}}}s(x,"cronRefineCases");export{x as a};

package/dist/{chunk-7MTYUTUN.js → chunk-KR36ZDGN.js}

@@ -1,4 +1,4 @@
-import{f as u}from"./chunk-FBSZ6KCM.js";import{a as o}from"./chunk-ZLLER3HP.js";var l=`You are a meta-judge evaluating the quality of an eval run \u2014 not the code output, but whether the test cases and grading themselves are rigorous and meaningful.
+import{f as u}from"./chunk-4GOXEEXZ.js";import{a as o}from"./chunk-ZLLER3HP.js";var l=`You are a meta-judge evaluating the quality of an eval run \u2014 not the code output, but whether the test cases and grading themselves are rigorous and meaningful.
 
 You will receive a summary of eval cases, their expectations, and grading results.
 

package/dist/{chunk-O4K4M5G6.js → chunk-LJE5J3EW.js}

@@ -1 +1 @@
-import{d as h,e as g}from"./chunk-RSVNQ2XI.js";import{e as p,g as f}from"./chunk-GIXHG5FK.js";import{a as o}from"./chunk-ZLLER3HP.js";var u="openCandidate";function K(n,s,a){let t=(n??{})[u];if(!t||typeof t!="object"||Array.isArray(t))return null;let e=t;return e.executionKey!==a||typeof e.content!="string"||!e.content||typeof e.hash!="string"||!e.hash||typeof e.priorHash!="string"||e.priorHash!==s?null:{...a?{executionKey:a}:{},...typeof e.observedKey=="string"?{observedKey:e.observedKey}:{},content:e.content,hash:e.hash,priorHash:e.priorHash,plannedPairs:typeof e.plannedPairs=="number"?e.plannedPairs:0,updatedAt:typeof e.updatedAt=="string"?e.updatedAt:""}}o(K,"resolveOpenCandidate");function C(n){let{plannedPairs:s,carriedPairs:a,liveCases:t}=n;if(t<=0)return 0;let e=s-a;return e<=0?0:Math.ceil(e/t)}o(C,"owedPasses");var d=100;async function x(n){let{evalTargetId:s,candidateContentHash:a}=n,t=[];for(let e=0;;e+=d){let i=await p(`/api/eval-targets/${s}/runs?candidateContentHash=${encodeURIComponent(a)}&limit=${d}&offset=${e}`);if(!Array.isArray(i)||i.length===0||(t.push(...i.filter(r=>{if(r.outputs?.executionKey!==n.executionKey)return!1;if(!n.executionKey)return!0;if(!n.observedKey)return!1;let l=h([r]);if(!l)return!1;let[y,c]=JSON.parse(l),[b,m]=JSON.parse(n.observedKey);return y===b&&(c===null||c===m)})),i.length<d))break}return g(t)}o(x,"fetchCarriedPairs");async function A(n){let{evalTargetId:s,meta:a,blob:t,log:e,tag:i}=n;try{let r={...a};t?r[u]=t:delete r[u],await f(`/api/eval-targets/${s}`,{metadata:r}),e(t?`${i}: banked candidate ${t.hash.slice(0,8)} \u2014 the next run resumes instead of restarting`:`${i}: cleared the banked candidate (this cycle reached a decision)`)}catch(r){e(`${i}: could not update the candidate bank (${r instanceof Error?r.message:String(r)})`)}}o(A,"bankOpenCandidate");function O(n){return{...n.executionKey?{executionKey:n.executionKey}:{},...n.observedKey?{observedKey:n.observedKey}:{},content:n.content,hash:n.hash,priorHash:n.priorHash,plannedPairs:n.plannedPairs,updatedAt:n.now.toISOString()}}o(O,"buildOpenCandidate");export{u as a,K as b,C as c,x as d,A as e,O as f};
+import{d as h,e as g}from"./chunk-YXMAGC53.js";import{e as p,g as f}from"./chunk-GIXHG5FK.js";import{a as o}from"./chunk-ZLLER3HP.js";var u="openCandidate";function K(n,s,a){let t=(n??{})[u];if(!t||typeof t!="object"||Array.isArray(t))return null;let e=t;return e.executionKey!==a||typeof e.content!="string"||!e.content||typeof e.hash!="string"||!e.hash||typeof e.priorHash!="string"||e.priorHash!==s?null:{...a?{executionKey:a}:{},...typeof e.observedKey=="string"?{observedKey:e.observedKey}:{},content:e.content,hash:e.hash,priorHash:e.priorHash,plannedPairs:typeof e.plannedPairs=="number"?e.plannedPairs:0,updatedAt:typeof e.updatedAt=="string"?e.updatedAt:""}}o(K,"resolveOpenCandidate");function C(n){let{plannedPairs:s,carriedPairs:a,liveCases:t}=n;if(t<=0)return 0;let e=s-a;return e<=0?0:Math.ceil(e/t)}o(C,"owedPasses");var d=100;async function x(n){let{evalTargetId:s,candidateContentHash:a}=n,t=[];for(let e=0;;e+=d){let i=await p(`/api/eval-targets/${s}/runs?candidateContentHash=${encodeURIComponent(a)}&limit=${d}&offset=${e}`);if(!Array.isArray(i)||i.length===0||(t.push(...i.filter(r=>{if(r.outputs?.executionKey!==n.executionKey)return!1;if(!n.executionKey)return!0;if(!n.observedKey)return!1;let l=h([r]);if(!l)return!1;let[y,c]=JSON.parse(l),[b,m]=JSON.parse(n.observedKey);return y===b&&(c===null||c===m)})),i.length<d))break}return g(t)}o(x,"fetchCarriedPairs");async function A(n){let{evalTargetId:s,meta:a,blob:t,log:e,tag:i}=n;try{let r={...a};t?r[u]=t:delete r[u],await f(`/api/eval-targets/${s}`,{metadata:r}),e(t?`${i}: banked candidate ${t.hash.slice(0,8)} \u2014 the next run resumes instead of restarting`:`${i}: cleared the banked candidate (this cycle reached a decision)`)}catch(r){e(`${i}: could not update the candidate bank (${r instanceof Error?r.message:String(r)})`)}}o(A,"bankOpenCandidate");function O(n){return{...n.executionKey?{executionKey:n.executionKey}:{},...n.observedKey?{observedKey:n.observedKey}:{},content:n.content,hash:n.hash,priorHash:n.priorHash,plannedPairs:n.plannedPairs,updatedAt:n.now.toISOString()}}o(O,"buildOpenCandidate");export{u as a,K as b,C as c,x as d,A as e,O as f};

package/dist/{chunk-PXYK4MBX.js → chunk-LLXC5SRC.js}

@@ -1 +1 @@
-import{a as i}from"./chunk-FS3TAQAE.js";import{h as s}from"./chunk-GIXHG5FK.js";import{a as n}from"./chunk-ZLLER3HP.js";async function l(a,c){let{mirrorId:g,evalTargetId:p,config:r}=a,e=n(o=>c.logger.info(o),"log");await i(r,p,e);let t={type:"conversation",conversationId:r.conversationId};return await s(`/api/eval-jobs/${g}/state`,{result:t}).catch(o=>{e(`result persist failed: ${o instanceof Error?o.message:String(o)}`)}),t}n(l,"processConversation");export{l as a};
+import{a as i}from"./chunk-JOXTWLLK.js";import{h as s}from"./chunk-GIXHG5FK.js";import{a as n}from"./chunk-ZLLER3HP.js";async function l(a,c){let{mirrorId:g,evalTargetId:p,config:r}=a,e=n(o=>c.logger.info(o),"log");await i(r,p,e);let t={type:"conversation",conversationId:r.conversationId};return await s(`/api/eval-jobs/${g}/state`,{result:t}).catch(o=>{e(`result persist failed: ${o instanceof Error?o.message:String(o)}`)}),t}n(l,"processConversation");export{l as a};

package/dist/chunk-N2PP2A7M.js

@@ -0,0 +1 @@
+import{b as R,c as L,f as s,g as I,i as h}from"./chunk-XXPWPKTD.js";import{c as ne}from"./chunk-5TM7AOH2.js";import{b as re}from"./chunk-IEQDLURA.js";import{d as te,e as oe,f as se}from"./chunk-KINIFUZQ.js";import{a as Y}from"./chunk-5PMTNNA6.js";import{a as O}from"./chunk-ZYGBABMZ.js";import{a as U}from"./chunk-IFIWDL7G.js";import{c as N}from"./chunk-FZMO7XSN.js";import{a as F}from"./chunk-LLXC5SRC.js";import{a as W}from"./chunk-4DA6L7KL.js";import{a as $}from"./chunk-WQFR2LOY.js";import{a as q}from"./chunk-Q7IDLOSP.js";import{b as J}from"./chunk-XIOYV5A7.js";import{b as V}from"./chunk-5RBLIVBF.js";import{a as K}from"./chunk-JSGXFD42.js";import{a as x}from"./chunk-CZBO3A6B.js";import{a as j}from"./chunk-JUBXWTJA.js";import{a as Q}from"./chunk-W6MMJRFZ.js";import{a as D}from"./chunk-PZS55PW2.js";import{a as ee}from"./chunk-QPAOX7P2.js";import{e as P}from"./chunk-CIKQXFQ2.js";import{b as G}from"./chunk-5Z6XXOJL.js";import{a as z}from"./chunk-CYCH2EQC.js";import{a as X}from"./chunk-VUNJGZPP.js";import{a as Z}from"./chunk-L562Y4RN.js";import{b as C}from"./chunk-PIVEJTNF.js";import{k as B}from"./chunk-DYET4EYU.js";import{d as S}from"./chunk-GPBN4ROE.js";import{b as M,e as T,f as E}from"./chunk-GIXHG5FK.js";import{a as l}from"./chunk-ZLLER3HP.js";var H=null,w=null,ce={process_eval:W,process_insights:$,process_launch:J,process_send_message:O,process_conversation:F,process_session_goal:U,distill_memories:G,memory_ab:K,process_run_cleanup:Y,cron_refine_cases:j,cron_retire_unused_skills:ee,cron_reconcile_attribution:x,cron_suggest_instructions:P,cron_refresh_skills:D,cron_refresh_pricing:Q,cron_eval_autorun:X,cron_auto_accept_suggestions:z,cron_instruction_training:Z,cron_instruction_evolution:B,process_judge_review:q,harbor_ab:N},ae=Object.keys(ce),me=new Set([]);function pe(e=process.env){return e.DATABASE_URL?ae:ae.filter(t=>!me.has(t))}l(pe,"availableCapabilities");var ge=3e4,_e=3e4,fe=3e4,ie=500,le=1e4,ve=`thin-agent-${process.pid}`,A=null;function ke(e,t,r,p){let d;try{let i=S();d={claude:i.claudeBin!=null,codex:i.codexBin!=null,pi:i.piBin!=null,codexModel:i.codexConfiguredModel}}catch{}let m;try{m=C()}catch{}let g=l(i=>E("/api/workers/heartbeat",{workerType:"cli-worker",clientName:ve,clientVersion:p,workerVersion:"0.1.0",currentJobId:null,inFlightCount:0,lastError:null,status:i,metadata:{capabilities:e,mode:"thin-agent",measurementProtocol:1,studyProtocol:1,memorySnapshots:1,boundedAutorun:1,llmEngines:d,harborReady:m,machine:H?.summary()}}).then(c=>{let u=c?.advisory;u&&u.message!==A?(A=u.message,s(r,{event:"worker_advisory",code:u.code,message:u.message})):u||(A=null)}).catch(c=>{s(r,{event:"presence_heartbeat_error",error:c instanceof Error?c.message:String(c)})}),"send");g("alive");let f=setInterval(()=>g("alive"),fe),a=null,n=l(()=>a||(clearInterval(f),a=Promise.race([g("shutting_down"),new Promise(i=>setTimeout(i,2e3).unref())]),a),"sayGoodbye");return t.addEventListener("abort",()=>{n()},{once:!0}),{shutdown:n}}l(ke,"startPresenceHeartbeat");async function be(e,t,r=S()){let p=[...r.claudeBin?["claude"]:[],...r.codexBin?["codex"]:[],...r.piBin?["pi"]:[]],d=`capabilities=${encodeURIComponent(e.join(","))}&waitMs=${t}&measurementProtocol=1&studyProtocol=1&memorySnapshots=1&boundedAutorun=1&measurementEngines=${encodeURIComponent(p.join(","))}`,m=await T(`/api/work/next?${d}`);return m==null||typeof m=="string"?null:m}l(be,"pollForWork");function he(e,t){return{logger:{info:l(r=>t(r),"info"),warn:l(r=>t(r),"warn"),error:l(r=>t(r),"error")},job:{attempts:e}}}l(he,"makeHelpers");async function we(e,t,r,p){let d=ce[e.kind],m=Date.now();if(s(r,{event:"claim",jobId:e.id,kind:e.kind,attempts:e.attempts}),!d){let n=`unknown kind: ${e.kind}`;s(r,{event:"fail",jobId:e.id,kind:e.kind,terminal:!0,error:n}),await E(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:n,terminal:!0}).catch(()=>{});return}let g=l(()=>{s(r,{event:"heartbeat",jobId:e.id}),h(`/api/work/${e.id}/heartbeat`,{leaseToken:e.leaseToken},{budgetMs:Math.min(L,Math.max(0,t-1e3)),signal:p,log:r,event:"heartbeat",jobId:e.id}).then(n=>{n.ok||s(r,{event:"heartbeat_error",jobId:e.id,attempts:n.attempts,error:n.error instanceof Error?n.error.message:String(n.error)})})},"beat"),f=setInterval(g,t);w={jobId:e.id,beat:g};let a;try{let n=he(e.attempts,r);a={kind:"complete",result:await d(e.payload,n)??{}}}catch(n){let i=H?.pausesSince(m)??[],c=se(i),u=n instanceof Error?n.message:String(n),v=V(n),o=!v&&oe(i);a={kind:"fail",message:c?`${c}; ${u}`:u,pausedDuringJob:!!c,terminal:v,hostSuspended:o}}try{if(a.kind==="complete"){s(r,{event:"complete",jobId:e.id,kind:e.kind,durationMs:Date.now()-m});let n=await h(`/api/work/${e.id}/complete`,{leaseToken:e.leaseToken,result:a.result},{budgetMs:R,signal:p,log:r,event:"complete_post",jobId:e.id});n.ok||s(r,{event:"complete_post_error",jobId:e.id,kind:e.kind,attempts:n.attempts,error:n.error instanceof Error?n.error.message:String(n.error)})}else{s(r,{event:"fail",jobId:e.id,kind:e.kind,durationMs:Date.now()-m,terminal:a.terminal,error:a.message,...a.pausedDuringJob?{pausedDuringJob:!0}:{},...a.hostSuspended?{hostSuspended:!0}:{}});let n=await h(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:a.message,terminal:a.terminal,hostSuspended:a.hostSuspended},{budgetMs:R,signal:p,log:r,event:"fail_post",jobId:e.id});n.ok||s(r,{event:"fail_post_error",jobId:e.id,attempts:n.attempts,error:n.error instanceof Error?n.error.message:String(n.error)})}}finally{clearInterval(f),w=null}}l(we,"runOne");var de=new Set(["process_eval","process_conversation","process_insights","process_session_goal","distill_memories","memory_ab","cron_suggest_instructions","cron_eval_autorun","cron_instruction_evolution","process_judge_review","harbor_ab"]);async function Ze(e){let t=e.log??(()=>{}),r=e.pollWaitMs??ge,p=e.heartbeatIntervalMs??_e,d=e.capabilities??pe();s(t,{event:"agent_start",capabilities:d,pollWaitMs:r,heartbeatIntervalMs:p});try{let o=re();o.removed.length>0&&s(t,{event:"eval_worktrees_swept",removed:o.removed.length})}catch(o){s(t,{event:"eval_worktree_sweep_failed",error:o instanceof Error?o.message:String(o)})}H=te({signal:e.signal,onPause:l(o=>{s(t,{event:"machine_pause",...o,jobId:w?.jobId??null}),w?.beat()},"onPause")});let m=ke(d,e.signal,t,e.version);ne({signal:e.signal,log:t}).catch(o=>{s(t,{event:"tt_heartbeat_fatal",error:o instanceof Error?o.message:String(o)})});let g=5*6e4,f=0,a=l(()=>{if(!e.readInstalledVersion||!e.version)return!1;let o=Date.now();if(o-f<g)return!1;f=o;let b=e.readInstalledVersion();return!!(b&&b!==e.version)},"versionDrifted"),n=new AbortController,i=n.signal;e.signal.aborted?n.abort():e.signal.addEventListener("abort",()=>n.abort(),{once:!0});let c=l(async(o,b)=>{let _=ie;for(;!i.aborted;){if(a()){s(t,{event:"agent_version_drift",bootVersion:e.version,installedVersion:e.readInstalledVersion?.()??null}),n.abort(),e.onVersionDrift?.();break}let y=null;try{y=await be([...b],r)}catch(k){let ue=k instanceof M?k.statusCode:void 0;if(s(t,{event:"poll_error",lane:o,status:ue,error:k instanceof Error?k.message:String(k)}),!await I(_,i))break;_=Math.min(_*2,le);continue}if(!y){if(s(t,{event:"poll_204",lane:o}),!await I(_,i))break;_=Math.min(_*2,le);continue}_=ie,await we(y,p,t,i)}},"runLane"),u=d.filter(o=>de.has(o)),v=d.filter(o=>!de.has(o));u.length>0&&v.length>0?(s(t,{event:"agent_lanes",heavy:u.length,fast:v.length}),await Promise.all([c("heavy",u),c("fast",v)])):await c("single",d),await m.shutdown(),s(t,{event:"agent_stop"})}l(Ze,"runAgentLoop");export{ce as a,ae as b,pe as c,be as d,Ze as e};

package/dist/{chunk-KOYHJFB3.js → chunk-PDXHRF4E.js}

@@ -1,4 +1,4 @@
-import{f as p}from"./chunk-FBSZ6KCM.js";import{a as c}from"./chunk-ZLLER3HP.js";var y=`You are an expert at improving AI coding instructions, skills, and prompts. You receive eval results showing how well the current content performs, and your job is to propose specific improvements.
+import{f as p}from"./chunk-4GOXEEXZ.js";import{a as c}from"./chunk-ZLLER3HP.js";var y=`You are an expert at improving AI coding instructions, skills, and prompts. You receive eval results showing how well the current content performs, and your job is to propose specific improvements.
 
 You will receive:
 1. The current content of the eval target

package/dist/{chunk-F5PJN2EY.js → chunk-Q7IDLOSP.js}

@@ -1 +1 @@
-import{c as d,d as t}from"./chunk-6AAAHV2B.js";import{a as r}from"./chunk-ZLLER3HP.js";async function g(o,i){let{evalTargetId:s}=o;try{let e=await t({targetId:s,engineFlag:"auto",skipReviewed:!0,log:r(a=>i.logger.info(`[judge-review] ${a}`),"log"),warn:r(a=>i.logger.warn(`[judge-review] ${a}`),"warn")});return{evalTargetId:s,reviewerModel:e.reviewerModel,comparedPairs:e.report.pairs,disagreements:e.report.disagreements.length,persisted:e.persisted,alreadyReviewed:e.alreadyReviewed,deferredPartialRuns:e.deferredPartialRuns}}catch(e){if(e instanceof d)return i.logger.info(`[judge-review] skipped (${e.reason}): ${e.message}`),{evalTargetId:s,skipped:e.reason};throw e}}r(g,"processJudgeReview");export{g as a};
+import{c as d,d as t}from"./chunk-3H44B6PB.js";import{a as r}from"./chunk-ZLLER3HP.js";async function g(o,i){let{evalTargetId:s}=o;try{let e=await t({targetId:s,engineFlag:"auto",skipReviewed:!0,log:r(a=>i.logger.info(`[judge-review] ${a}`),"log"),warn:r(a=>i.logger.warn(`[judge-review] ${a}`),"warn")});return{evalTargetId:s,reviewerModel:e.reviewerModel,comparedPairs:e.report.pairs,disagreements:e.report.disagreements.length,persisted:e.persisted,alreadyReviewed:e.alreadyReviewed,deferredPartialRuns:e.deferredPartialRuns}}catch(e){if(e instanceof d)return i.logger.info(`[judge-review] skipped (${e.reason}): ${e.message}`),{evalTargetId:s,skipped:e.reason};throw e}}r(g,"processJudgeReview");export{g as a};

package/dist/{chunk-H2FMAJUJ.js → chunk-QFTAU33Y.js}

@@ -1 +1 @@
-import{b as $}from"./chunk-VBXL7XO7.js";import{e as w,f as k}from"./chunk-GIXHG5FK.js";import{a as u}from"./chunk-ZLLER3HP.js";function h(e){if(e.length===0)return{mean:0,stddev:0,min:0,max:0};let n=e.reduce((s,a)=>s+a,0)/e.length,o=e.reduce((s,a)=>s+(a-n)**2,0)/e.length;return{mean:Math.round(n*1e3)/1e3,stddev:Math.round(Math.sqrt(o)*1e3)/1e3,min:Math.min(...e),max:Math.max(...e)}}u(h,"computeStats");var x="with_target",E="without_target";function _(e){let n=e.grading.summary?.pass_rate;return typeof n=="number"&&n>=1}u(_,"fullPass");function b(e){return e.grading.summary?.pass_rate!=null}u(b,"hasGrading");function R(e){let n=e.timing.total_tokens;return typeof n=="number"&&n>0?n:void 0}u(R,"tokensOf");function v(e){let n=e.timing.duration_ms;return typeof n=="number"&&!isNaN(n)?n/1e3:void 0}u(v,"secondsOf");function y(e,n){let o={};for(let s of e)(o[n(s)]??=[]).push(s);return o}u(y,"groupBy");function C(e,n){let o=y(e.filter(b),i=>i.evalId),s=y(n.filter(b),i=>i.evalId),a=u((i,l)=>i.runNumber-l.runNumber,"byRun"),t=[];for(let i of Object.keys(o)){let l=o[i].sort(a),d=(s[i]??[]).sort(a);for(let r=0;r<Math.min(l.length,d.length);r++){let c={withScore:_(l[r])?1:0,withoutScore:_(d[r])?1:0},f=R(l[r]),g=R(d[r]);f!=null&&g!=null&&(c.withCost=f,c.withoutCost=g);let p=v(l[r]),m=v(d[r]);p!=null&&m!=null&&(c.withLatency=p,c.withoutLatency=m),t.push(c)}}return t}u(C,"pairRunsByEval");function T(e){let n=e.filter(r=>r.status==="completed");if(n.length===0)return{runSummary:{},verdict:null,notes:["No completed runs found"],totalRuns:0};let o=y(n,r=>r.configuration),s={},a=[];for(let[r,c]of Object.entries(o)){let f=c.filter(b).map(m=>m.grading.summary.pass_rate),g=c.map(v).filter(m=>m!=null),p=c.map(R).filter(m=>m!=null);s[r]={},f.length>0&&(s[r].pass_rate=h(f)),g.length>0&&(s[r].time_seconds=h(g)),p.length>0&&(s[r].tokens=h(p))}let t=null,i=o[x]??[],l=o[E]??[],d=C(i,l);if(d.length>0){if(t=$(d),!t.sufficient)a.push(t.insufficientReason??"Insufficient data for a significance verdict.");else if(t.significant){let r=t.delta>=0?"improves":"reduces",c=t.pValue!=null?`, McNemar p=${t.pValue.toFixed(4)}`:"";a.push(`Target ${r} the pass rate by ${Math.abs(t.delta*100).toFixed(0)}% (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}${c}).`)}else a.push(`No significant effect on pass rate (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}).`);t.costDelta!=null&&Math.round(t.costDelta)!==0&&a.push(`Token cost delta: ${t.costDelta>=0?"+":""}${Math.round(t.costDelta)} tokens/task.`),t.latencyDelta!=null&&Math.abs(t.latencyDelta)>1&&a.push(`Target ${t.latencyDelta>0?"adds":"saves"} ${Math.abs(t.latencyDelta).toFixed(1)}s/task.`)}for(let r of Object.values(s))if(r.pass_rate&&r.pass_rate.stddev>.2){a.push(`High pass-rate variance (stddev=${r.pass_rate.stddev.toFixed(2)}) \u2014 results may be unreliable.`);break}return{runSummary:s,verdict:t,notes:a,totalRuns:n.length}}u(T,"summarizeRuns");async function S(e,n){let o=n?`&iteration=${n}`:"",s=100,a=[];for(let t=0;t<1e5;t+=s){let i=await w(`/api/eval-targets/${e}/runs?limit=${s}&offset=${t}&activeCasesOnly=1${o}`);if(a.push(...i),i.length<s)break}return T(a)}u(S,"aggregateBenchmark");async function B(e,n,o){return k(`/api/eval-targets/${e}/benchmarks`,{iteration:n,runSummary:o.runSummary,notes:o.notes})}u(B,"postBenchmark");export{T as a,S as b,B as c};
+import{b as $}from"./chunk-HT6R6KR3.js";import{e as w,f as k}from"./chunk-GIXHG5FK.js";import{a as u}from"./chunk-ZLLER3HP.js";function h(e){if(e.length===0)return{mean:0,stddev:0,min:0,max:0};let n=e.reduce((s,a)=>s+a,0)/e.length,o=e.reduce((s,a)=>s+(a-n)**2,0)/e.length;return{mean:Math.round(n*1e3)/1e3,stddev:Math.round(Math.sqrt(o)*1e3)/1e3,min:Math.min(...e),max:Math.max(...e)}}u(h,"computeStats");var x="with_target",E="without_target";function _(e){let n=e.grading.summary?.pass_rate;return typeof n=="number"&&n>=1}u(_,"fullPass");function b(e){return e.grading.summary?.pass_rate!=null}u(b,"hasGrading");function R(e){let n=e.timing.total_tokens;return typeof n=="number"&&n>0?n:void 0}u(R,"tokensOf");function v(e){let n=e.timing.duration_ms;return typeof n=="number"&&!isNaN(n)?n/1e3:void 0}u(v,"secondsOf");function y(e,n){let o={};for(let s of e)(o[n(s)]??=[]).push(s);return o}u(y,"groupBy");function C(e,n){let o=y(e.filter(b),i=>i.evalId),s=y(n.filter(b),i=>i.evalId),a=u((i,l)=>i.runNumber-l.runNumber,"byRun"),t=[];for(let i of Object.keys(o)){let l=o[i].sort(a),d=(s[i]??[]).sort(a);for(let r=0;r<Math.min(l.length,d.length);r++){let c={withScore:_(l[r])?1:0,withoutScore:_(d[r])?1:0},f=R(l[r]),g=R(d[r]);f!=null&&g!=null&&(c.withCost=f,c.withoutCost=g);let p=v(l[r]),m=v(d[r]);p!=null&&m!=null&&(c.withLatency=p,c.withoutLatency=m),t.push(c)}}return t}u(C,"pairRunsByEval");function T(e){let n=e.filter(r=>r.status==="completed");if(n.length===0)return{runSummary:{},verdict:null,notes:["No completed runs found"],totalRuns:0};let o=y(n,r=>r.configuration),s={},a=[];for(let[r,c]of Object.entries(o)){let f=c.filter(b).map(m=>m.grading.summary.pass_rate),g=c.map(v).filter(m=>m!=null),p=c.map(R).filter(m=>m!=null);s[r]={},f.length>0&&(s[r].pass_rate=h(f)),g.length>0&&(s[r].time_seconds=h(g)),p.length>0&&(s[r].tokens=h(p))}let t=null,i=o[x]??[],l=o[E]??[],d=C(i,l);if(d.length>0){if(t=$(d),!t.sufficient)a.push(t.insufficientReason??"Insufficient data for a significance verdict.");else if(t.significant){let r=t.delta>=0?"improves":"reduces",c=t.pValue!=null?`, McNemar p=${t.pValue.toFixed(4)}`:"";a.push(`Target ${r} the pass rate by ${Math.abs(t.delta*100).toFixed(0)}% (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}${c}).`)}else a.push(`No significant effect on pass rate (95% CI ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%, n=${t.nWith}).`);t.costDelta!=null&&Math.round(t.costDelta)!==0&&a.push(`Token cost delta: ${t.costDelta>=0?"+":""}${Math.round(t.costDelta)} tokens/task.`),t.latencyDelta!=null&&Math.abs(t.latencyDelta)>1&&a.push(`Target ${t.latencyDelta>0?"adds":"saves"} ${Math.abs(t.latencyDelta).toFixed(1)}s/task.`)}for(let r of Object.values(s))if(r.pass_rate&&r.pass_rate.stddev>.2){a.push(`High pass-rate variance (stddev=${r.pass_rate.stddev.toFixed(2)}) \u2014 results may be unreliable.`);break}return{runSummary:s,verdict:t,notes:a,totalRuns:n.length}}u(T,"summarizeRuns");async function S(e,n){let o=n?`&iteration=${n}`:"",s=100,a=[];for(let t=0;t<1e5;t+=s){let i=await w(`/api/eval-targets/${e}/runs?limit=${s}&offset=${t}&activeCasesOnly=1${o}`);if(a.push(...i),i.length<s)break}return T(a)}u(S,"aggregateBenchmark");async function B(e,n,o){return k(`/api/eval-targets/${e}/benchmarks`,{iteration:n,runSummary:o.runSummary,notes:o.notes})}u(B,"postBenchmark");export{T as a,S as b,B as c};

package/dist/{chunk-EBWV4SUS.js → chunk-RDL4HXFL.js}

@@ -1,4 +1,4 @@
-import{a as f,b as M}from"./chunk-Q776YUBD.js";import{e as _,f as S}from"./chunk-X6CF5KTW.js";import{a as o}from"./chunk-ZLLER3HP.js";import*as m from"node:fs/promises";import*as x from"node:path";import{eq as j}from"drizzle-orm";import{randomBytes as Y}from"node:crypto";var A="#0d9488",h="#1a1a1a",$="#666666",C="#f5f5f7",N="#ffffff",D="#e5e5e9",I=["Added","Changed","Fixed","Removed","Deprecated","Security"];function U(e){let t=["January","February","March","April","May","June","July","August","September","October","November","December"],[n,r,a]=e.split("-").map(d=>parseInt(d,10));return!n||!r||!a||r<1||r>12?e:`${t[r-1]} ${a}, ${n}`}o(U,"formatIsoDate");function l(e){return e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#39;")}o(l,"escapeHtml");function z(e){let t=[];for(let n of I){let r=e[n];r&&r.length>0&&t.push({name:n,items:r})}return t}o(z,"orderedSections");function v(e){let{version:t,date:n,sections:r,unsubscribeUrl:a,productUrl:d,recipientName:u}=e,c=U(n),s=`What's new in RuleMetric \u2014 ${c}`,y=u?`Hi ${u},`:"Hi there,",i=z(r),p=new Date().getUTCFullYear(),b=i.map(({name:R,items:E})=>{let w=E.map(O=>`              <li style="margin:0 0 6px 0;font-size:15px;line-height:1.5;color:${h};">${l(O)}</li>`).join(`
+import{a as f,b as M}from"./chunk-V576KKH6.js";import{e as _,f as S}from"./chunk-X6CF5KTW.js";import{a as o}from"./chunk-ZLLER3HP.js";import*as m from"node:fs/promises";import*as x from"node:path";import{eq as j}from"drizzle-orm";import{randomBytes as Y}from"node:crypto";var A="#0d9488",h="#1a1a1a",$="#666666",C="#f5f5f7",N="#ffffff",D="#e5e5e9",I=["Added","Changed","Fixed","Removed","Deprecated","Security"];function U(e){let t=["January","February","March","April","May","June","July","August","September","October","November","December"],[n,r,a]=e.split("-").map(d=>parseInt(d,10));return!n||!r||!a||r<1||r>12?e:`${t[r-1]} ${a}, ${n}`}o(U,"formatIsoDate");function l(e){return e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#39;")}o(l,"escapeHtml");function z(e){let t=[];for(let n of I){let r=e[n];r&&r.length>0&&t.push({name:n,items:r})}return t}o(z,"orderedSections");function v(e){let{version:t,date:n,sections:r,unsubscribeUrl:a,productUrl:d,recipientName:u}=e,c=U(n),s=`What's new in RuleMetric \u2014 ${c}`,y=u?`Hi ${u},`:"Hi there,",i=z(r),p=new Date().getUTCFullYear(),b=i.map(({name:R,items:E})=>{let w=E.map(O=>`              <li style="margin:0 0 6px 0;font-size:15px;line-height:1.5;color:${h};">${l(O)}</li>`).join(`
 `);return`            <h3 style="margin:24px 0 8px 0;font-size:16px;line-height:1.3;font-weight:600;color:${h};">${R}</h3>
             <ul style="margin:0 0 8px 0;padding-left:20px;">
 ${w}

package/dist/{chunk-ZK3ACE4T.js → chunk-T7HZZ4DJ.js}

@@ -1 +1 @@
-import{b as s,c as a}from"./chunk-5MXZJWQS.js";import{f as c}from"./chunk-FBSZ6KCM.js";import{f as u}from"./chunk-XQYAVSUQ.js";import{a as m}from"./chunk-ZLLER3HP.js";import{mkdtempSync as p,rmSync as d}from"node:fs";import{join as f}from"node:path";import{tmpdir as l}from"node:os";async function g(n,o,i){let r=u.parse(n),e=p(f(l(),"rulemetric-measurement-generation-"));try{let t=await c({engine:r.engine,model:r.model??void 0,systemPrompt:o,prompt:i,workDir:e,timeout:12e4});if(t.error||t.exitCode!==0||!t.output.trim())throw new Error(`Selected ${r.engine} generator failed: ${t.error??`exit ${t.exitCode}, empty output`}`);return t.output}finally{d(e,{recursive:!0,force:!0})}}m(g,"runMeasurementText");async function S(n,o,i){let r=await g(n,o,i),e=s(a(r)??r);if(!e.ok||!e.value||Array.isArray(e.value)||typeof e.value!="object")throw new Error(`Selected ${n.engine} generator returned invalid JSON`);return e.value}m(S,"runMeasurementJson");export{g as a,S as b};
+import{b as s,c as a}from"./chunk-5MXZJWQS.js";import{f as c}from"./chunk-4GOXEEXZ.js";import{f as u}from"./chunk-XQYAVSUQ.js";import{a as m}from"./chunk-ZLLER3HP.js";import{mkdtempSync as p,rmSync as d}from"node:fs";import{join as f}from"node:path";import{tmpdir as l}from"node:os";async function g(n,o,i){let r=u.parse(n),e=p(f(l(),"rulemetric-measurement-generation-"));try{let t=await c({engine:r.engine,model:r.model??void 0,systemPrompt:o,prompt:i,workDir:e,timeout:12e4});if(t.error||t.exitCode!==0||!t.output.trim())throw new Error(`Selected ${r.engine} generator failed: ${t.error??`exit ${t.exitCode}, empty output`}`);return t.output}finally{d(e,{recursive:!0,force:!0})}}m(g,"runMeasurementText");async function S(n,o,i){let r=await g(n,o,i),e=s(a(r)??r);if(!e.ok||!e.value||Array.isArray(e.value)||typeof e.value!="object")throw new Error(`Selected ${n.engine} generator returned invalid JSON`);return e.value}m(S,"runMeasurementJson");export{g as a,S as b};

package/dist/{chunk-LB24UKHZ.js → chunk-TWPD3ICV.js}

@@ -1 +1 @@
-import{a as m,c as d}from"./chunk-VBXL7XO7.js";import{a as s}from"./chunk-ZLLER3HP.js";var h=4,i=1,_=2,f=.0641;function x(e,r=f){return!(e>0)||!(r>=0)?null:Math.sqrt((m+d)**2*r/e)}s(x,"mdeForPairs");function w(e,r=f){if(!(e>0))throw new Error("pairsForMde: mde must be > 0");if(!(r>=0))throw new Error("pairsForMde: variance must be >= 0");return Math.max(1,Math.ceil((m+d)**2*r/e**2))}s(w,"pairsForMde");var p=10,c=128;function R(e){let r=e.autoEvolvePairs;if(r==null)return{minPairs:p,configured:!1};let a=typeof r=="number"?r:typeof r=="string"?Number(r):NaN;if(!Number.isFinite(a)||a<1)return{minPairs:p,configured:!1,note:`metadata.autoEvolvePairs=${JSON.stringify(r)} is not a positive number \u2014 using the default ${p}`};let n=Math.floor(a);return{minPairs:Math.min(c,Math.max(10,n)),configured:!0,...n>c?{note:`metadata.autoEvolvePairs=${n} clamped to the ${c}-pair ceiling`}:n<10?{note:`metadata.autoEvolvePairs=${n} raised to ${10} \u2014 a controlled verdict is refused below ${10} pairs, so a smaller batch buys a certain refusal. Use metadata.autoEvolvePaused to stop the spend instead.`}:{}}}s(R,"readPairBudget");function $(e){let r=e.autoEvolvePaused;return r===!0||r==="true"}s($,"isEvolutionPaused");function O(e){let{liveCases:r,passes:a,variance:n}=e,t=Math.max(0,r)*Math.max(0,a),o=t*2,l=t*2;return{liveCases:r,passes:a,plannedPairs:t,proposalCalls:i,executorCalls:o,graderCalls:l,totalCalls:i+o+l,maxCalls:i+o+l*_,mde:x(t,n)}}s(O,"estimateEvolutionCost");function L(e){let{runs:r,plannedPairs:a,usablePairs:n,passes:t,liveCases:o,wallMs:l,variance:g}=e,b=e.proposalCalls??i,v=e.verdictPairs??n,C=r.length,E=r.filter(P=>P.status==="completed"&&P.grading?.judge_model!=null).length;return{proposalCalls:b,executorCalls:C,graderCalls:E,modelCalls:b+C+E,plannedPairs:a,usablePairs:n,verdictPairs:v,passes:t,liveCases:o,mde:x(v,g),wallMs:l}}s(L,"observedEvolutionCost");function A(e){return e==null?"n/a":`${(e*100).toFixed(1)}pp`}s(A,"pp");function D(e){return`${e.liveCases} case(s) x ${e.passes} pass(es) = ${e.plannedPairs} planned pair(s), ~${e.totalCalls} model calls (up to ${e.maxCalls}) \u2014 resolves an effect of +/-${A(e.mde)} at 80% power`}s(D,"describeEvolutionCost");export{h as a,i as b,_ as c,f as d,x as e,w as f,p as g,c as h,R as i,$ as j,O as k,L as l,A as m,D as n};
+import{a as m,c as d}from"./chunk-HT6R6KR3.js";import{a as s}from"./chunk-ZLLER3HP.js";var h=4,i=1,_=2,f=.0641;function x(e,r=f){return!(e>0)||!(r>=0)?null:Math.sqrt((m+d)**2*r/e)}s(x,"mdeForPairs");function w(e,r=f){if(!(e>0))throw new Error("pairsForMde: mde must be > 0");if(!(r>=0))throw new Error("pairsForMde: variance must be >= 0");return Math.max(1,Math.ceil((m+d)**2*r/e**2))}s(w,"pairsForMde");var p=10,c=128;function R(e){let r=e.autoEvolvePairs;if(r==null)return{minPairs:p,configured:!1};let a=typeof r=="number"?r:typeof r=="string"?Number(r):NaN;if(!Number.isFinite(a)||a<1)return{minPairs:p,configured:!1,note:`metadata.autoEvolvePairs=${JSON.stringify(r)} is not a positive number \u2014 using the default ${p}`};let n=Math.floor(a);return{minPairs:Math.min(c,Math.max(10,n)),configured:!0,...n>c?{note:`metadata.autoEvolvePairs=${n} clamped to the ${c}-pair ceiling`}:n<10?{note:`metadata.autoEvolvePairs=${n} raised to ${10} \u2014 a controlled verdict is refused below ${10} pairs, so a smaller batch buys a certain refusal. Use metadata.autoEvolvePaused to stop the spend instead.`}:{}}}s(R,"readPairBudget");function $(e){let r=e.autoEvolvePaused;return r===!0||r==="true"}s($,"isEvolutionPaused");function O(e){let{liveCases:r,passes:a,variance:n}=e,t=Math.max(0,r)*Math.max(0,a),o=t*2,l=t*2;return{liveCases:r,passes:a,plannedPairs:t,proposalCalls:i,executorCalls:o,graderCalls:l,totalCalls:i+o+l,maxCalls:i+o+l*_,mde:x(t,n)}}s(O,"estimateEvolutionCost");function L(e){let{runs:r,plannedPairs:a,usablePairs:n,passes:t,liveCases:o,wallMs:l,variance:g}=e,b=e.proposalCalls??i,v=e.verdictPairs??n,C=r.length,E=r.filter(P=>P.status==="completed"&&P.grading?.judge_model!=null).length;return{proposalCalls:b,executorCalls:C,graderCalls:E,modelCalls:b+C+E,plannedPairs:a,usablePairs:n,verdictPairs:v,passes:t,liveCases:o,mde:x(v,g),wallMs:l}}s(L,"observedEvolutionCost");function A(e){return e==null?"n/a":`${(e*100).toFixed(1)}pp`}s(A,"pp");function D(e){return`${e.liveCases} case(s) x ${e.passes} pass(es) = ${e.plannedPairs} planned pair(s), ~${e.totalCalls} model calls (up to ${e.maxCalls}) \u2014 resolves an effect of +/-${A(e.mde)} at 80% power`}s(D,"describeEvolutionCost");export{h as a,i as b,_ as c,f as d,x as e,w as f,p as g,c as h,R as i,$ as j,O as k,L as l,A as m,D as n};