@rulemetric/cli 0.17.0 → 0.17.2

package/dist/chunk-ZJABRHIP.js

@@ -0,0 +1 @@
+import{e as $,j as P,k as T,l as A,m as I,n as v,o as _,p as y,q as O,r as C,s as L,t as M}from"./chunk-XFOG2PDJ.js";import{b as x,c as D}from"./chunk-SIGZEQ3K.js";import{b as R,e as E,f as b}from"./chunk-5VWVQCNR.js";import{a as c}from"./chunk-ZLLER3HP.js";import{join as q}from"node:path";import{pathToFileURL as F}from"node:url";import{scriptsDir as z}from"@rulemetric/hooks/scripts-dir";var N=2e7,l=class extends Error{constructor(t,r,u,a,d){super(r,d instanceof Error?{cause:d}:void 0);this.phase=t;this.step=u;this.receiptDir=a;this.name="ScopedOutcomeError"}static{c(this,"ScopedOutcomeError")}};function te(e,o,t){if(e instanceof l&&e.phase==="deliver"&&e.receiptDir)return{status:"undelivered",sessionId:o,studyId:t,receiptDir:e.receiptDir,step:e.step,reason:e.message};let r=e instanceof l?e.step:"measure",u=e instanceof l?e.receiptDir:null;return{status:"unmeasured",sessionId:o,studyId:t,step:r,reason:e instanceof Error?e.message:String(e),receiptDir:u}}c(te,"scopedOutcomeFailure");async function U(){return import(F(q(z,"scoped-live.mjs")).href)}c(U,"loadCollector");async function Y(e,o){let{confirmation:t}=await E(`/api/scoped-live-confirmations/${e}/contract`,{signal:o?AbortSignal.any([o,AbortSignal.timeout(3e4)]):AbortSignal.timeout(3e4)});return P(e,t.protocol),t.protocol}c(Y,"stageScopedProtocol");async function G(e,o){let t=e.scopedConfirmation,r=$(t.id,o),u=v(r);if(u)return{dir:r,record:u,created:!1};let a=D(o,x(()=>null,()=>null)),d=a?C(a):null;if(!a||d===null||d>N)throw new l("collect","Completed actor transcript is missing or exceeds the capture bound","transcript",null);let s=A(r,a),n=T(t.id);if(n===null)try{n=await Y(t.id)}catch(i){throw new l("collect",`Frozen protocol unavailable after exit: ${i.message}`,"protocol",r,i)}let p;try{let i=await U();i.transcriptEvidence(s.path,o),p=i.collectScopedOutcome(n,t.taskId,o,e.projectPath,s.path)}catch(i){throw new l("collect",i.message,"measure",r,i)}let f=I(r,{confirmationId:t.id,taskId:t.taskId,externalSessionId:o,projectPath:e.projectPath,collectedAt:new Date().toISOString(),transcriptSha256:s.sha256,transcriptBytes:s.bytes,receipt:p});return{dir:r,...f}}c(G,"collectScopedReceipt");function V(e,{requestTimeoutMs:o,signal:t,deadlineAt:r,now:u=Date.now},a){if(t?.aborted)return Promise.reject(new Error(`${a} not attempted: ${k(t)}`));if(!o&&!t&&r===void 0)return e();let d=r===void 0?1/0:r-u();if(d<=0)return Promise.reject(new Error(`${a} not attempted: sweep deadline exhausted`));let s=Math.min(o??1/0,d),n=new AbortController;return new Promise((p,f)=>{let i=!1,g=c(w=>{i||(i=!0,clearTimeout(S),t?.removeEventListener("abort",m),w())},"finish"),m=c(()=>g(()=>{n.abort(t?.reason),f(new Error(`${a} interrupted: ${k(t)}`))}),"onAbort"),S=Number.isFinite(s)?setTimeout(()=>g(()=>{n.abort(),f(new Error(`${a} request exceeded ${Math.max(1,Math.round(s/1e3))}s deadline`))}),Math.max(1,s)):void 0;t?.addEventListener("abort",m,{once:!0}),e(n.signal).then(w=>g(()=>p(w)),w=>g(()=>f(w)))})}c(V,"bounded");function k(e){let o=e.reason;return o instanceof Error?o.message:typeof o=="string"?o:"aborted"}c(k,"abortReason");function h(e,o,t){return t?b(e,o,{signal:t}):b(e,o)}c(h,"postReceipt");async function B(e,o={}){let t=v(e);if(!t)throw new l("collect",`No receipt at ${e}; the outcome was never measured and cannot be reconstructed`,"measure",e);let r=_(e);if(r.completedAt)return{ledger:r,record:t};r.attempts+=1;let u=c(s=>r.completed.includes(s),"done"),a=c(s=>{u(s)||r.completed.push(s),y(e,r)},"mark"),d=c(async(s,n)=>{if(!u(s)){try{await V(n,o,s)}catch(p){throw r.lastError={at:new Date().toISOString(),step:s,message:p.message},y(e,r),new l("deliver",`${s} failed: ${p.message}`,s,e,p)}r.lastError=null,a(s)}},"run");return await d("session",async s=>{let n=await h("/api/sessions",{tool:"claude_code",externalSessionId:t.externalSessionId,projectPath:t.projectPath},s);r.sessionId=n.id}),await d("reimport",async s=>{await h(`/api/sessions/${r.sessionId}/reimport`,{tool:"claude_code",content:O(e).toString("utf8")},s)}),await d("verify",async s=>{await h(`/api/scoped-live-confirmations/${t.confirmationId}/verify`,t.receipt,s)}),await d("settle",async s=>{await h("/api/instruction-experiments/settle",{projectPath:t.projectPath},s)}),r.completedAt=new Date().toISOString(),y(e,r),{ledger:r,record:t}}c(B,"deliverScopedReceipt");async function re(e,o,t){let r=e.scopedConfirmation,{dir:u,created:a}=await G(e,o);return await B(u,{signal:t,requestTimeoutMs:3e4,deadlineAt:Date.now()+12e4}),{status:"recorded",sessionId:o,studyId:r.id,receiptDir:u,resumed:!a}}c(re,"finalizeScopedLaunch");var H={sweepBudgetMs:12e4,requestTimeoutMs:3e4,maxEntries:50};async function ne(e=()=>{},o={}){let{sweepBudgetMs:t,requestTimeoutMs:r,maxEntries:u}={...H,...o},a=o.now??Date.now,d=a(),s=o.signal,n={scanned:0,delivered:0,failed:0,unmeasured:0,pending:0,deferred:0,stopReason:"complete",entries:[]},p=[];for(let i of L()){if(n.scanned+=1,M(i)){n.entries.push({dir:i.dir,status:"already_delivered"});continue}if(!i.receipt){n.unmeasured+=1,n.entries.push({dir:i.dir,status:"unmeasured",reason:i.hasTranscript?"transcript preserved but no receipt was collected":"entry has neither transcript nor receipt"}),e(`scoped outcome ${i.confirmationId}/${i.externalSessionId}: unmeasured (no receipt); not reconstructed`);continue}p.push(i)}p.sort((i,g)=>(i.ledger?.lastError?.at??"").localeCompare(g.ledger?.lastError?.at??"")),n.pending=p.length;let f=0;for(let i of p){let g=a()-d;if(s?.aborted?n.stopReason="lease_lost":(g>=t||f>=u)&&(n.stopReason="budget_exhausted"),n.stopReason!=="complete"){n.deferred+=1,n.entries.push({dir:i.dir,status:"deferred",reason:n.stopReason});continue}f+=1;try{await B(i.dir,{requestTimeoutMs:r,signal:s,deadlineAt:d+t,now:a}),n.delivered+=1,n.pending-=1,n.entries.push({dir:i.dir,status:"delivered"}),e(`scoped outcome ${i.confirmationId}/${i.externalSessionId}: delivered`)}catch(m){n.failed+=1;let S=m instanceof l?m.step:"unknown",w=m instanceof Error?m.cause:void 0,j=w instanceof R?` (${w.statusCode})`:"";n.entries.push({dir:i.dir,status:"failed",step:S,reason:m.message}),e(`scoped outcome ${i.confirmationId}/${i.externalSessionId}: ${S} still failing${j}: ${m.message}`),s?.aborted&&(n.stopReason="lease_lost")}}return n.pending>0&&e(`scoped outcomes: ${n.pending} receipt(s) still pending upload (${n.deferred} deferred, stop=${n.stopReason})`),n}c(ne,"resumePendingScopedOutcomes");export{l as a,te as b,Y as c,G as d,B as e,re as f,H as g,ne as h};

package/dist/commands/antigravity/backfill.js

@@ -1,2 +1,2 @@
-import{a as v,b as u,e as y,g as d}from"../../chunk-BDQTVYYF.js";import{a as h}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as p}from"../../chunk-ZLLER3HP.js";import{Args as k,Flags as o}from"@oclif/core";import{existsSync as T}from"node:fs";import{homedir as P}from"node:os";import{join as I}from"node:path";var A=I(P(),".gemini","antigravity-ide","brain"),g=class x extends h{static{p(this,"AntigravityBackfill")}static description="Backfill historical events from an Antigravity conversation transcript into a RuleMetric session. Useful for seeding a conversation that existed before hooks were installed \u2014 the live hook intentionally only captures forward.";static examples=["<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --session 158db9aa-68ee-43fc-961b-f3a180578694","<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --dry-run"];static args={conversationId:k.string({description:"Antigravity conversationId (UUID) \u2014 the directory name under ~/.gemini/antigravity-ide/brain/",required:!0})};static flags={session:o.string({char:"s",description:"RuleMetric session UUID to POST events into. If omitted, resolved from the hook tmp file ($TMPDIR/rulemetric/<conversationId>) \u2014 fails if neither is present."}),transcript:o.string({description:"Path to the transcript.jsonl file. Defaults to ~/.gemini/antigravity-ide/brain/<conversationId>/.system_generated/logs/transcript.jsonl"}),"after-step":o.integer({description:"Only emit records with step_index > this value. Default: read from existing state file, else -1 (everything)."}),limit:o.integer({description:"Cap on the number of records to process this run."}),"dry-run":o.boolean({default:!1,description:"Classify records without POSTing. Reports counts only."}),"save-state":o.boolean({default:!0,allowNo:!0,description:"After backfill, persist the highest step_index so the live hook resumes from there."})};async run(){let{args:b,flags:e}=await this.parse(x),t=b.conversationId;d.loadEnvFile(),process.env.RULEMETRIC_API_URL||this.error("RULEMETRIC_API_URL is not set. Configure it via env or ~/.config/rulemetric/env.");let l=e.transcript??I(A,t,".system_generated","logs","transcript.jsonl");T(l)||this.error(`Transcript not found: ${l}`);let m=e.session??d.resolveRwSessionId(t);m||this.error(`No RuleMetric session resolved. Pass --session <uuid>, or run the Antigravity hook at least once so $TMPDIR/rulemetric/${t} is populated.`);let n=e["after-step"];n===void 0&&(n=d.load(t)?.lastStepIdx??-1);let i=v(l,n);e.limit!==void 0&&i.length>e.limit&&(i=i.slice(0,e.limit));let R=u(l);if(this.log(`Conversation ${t}: transcript max step = ${R}, after-step = ${n}, records to process = ${i.length}`),i.length===0){this.log("Nothing to backfill.");return}if(e["dry-run"]){let r=0,a=0,c={};for(let S of i){let f=(await import("../../lib/antigravity-transcript.js")).classifyRecord(S,0);f?(r++,c[f.eventType]=(c[f.eventType]??0)+1):a++}this.log(`Dry run: would emit ${r}, skip ${a}. By eventType: ${JSON.stringify(c)}`);return}let w=Date.now(),s=await y(i,m,t,n,{onProgress:p(({attempted:r,total:a})=>{(r%100===0||r===a)&&process.stderr.write(`  \u2026${r}/${a}
+import{a as v,b as u,e as y,g as d}from"../../chunk-HNPC5U3C.js";import{a as h}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as p}from"../../chunk-ZLLER3HP.js";import{Args as k,Flags as o}from"@oclif/core";import{existsSync as T}from"node:fs";import{homedir as P}from"node:os";import{join as I}from"node:path";var A=I(P(),".gemini","antigravity-ide","brain"),g=class x extends h{static{p(this,"AntigravityBackfill")}static description="Backfill historical events from an Antigravity conversation transcript into a RuleMetric session. Useful for seeding a conversation that existed before hooks were installed \u2014 the live hook intentionally only captures forward.";static examples=["<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --session 158db9aa-68ee-43fc-961b-f3a180578694","<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --dry-run"];static args={conversationId:k.string({description:"Antigravity conversationId (UUID) \u2014 the directory name under ~/.gemini/antigravity-ide/brain/",required:!0})};static flags={session:o.string({char:"s",description:"RuleMetric session UUID to POST events into. If omitted, resolved from the hook tmp file ($TMPDIR/rulemetric/<conversationId>) \u2014 fails if neither is present."}),transcript:o.string({description:"Path to the transcript.jsonl file. Defaults to ~/.gemini/antigravity-ide/brain/<conversationId>/.system_generated/logs/transcript.jsonl"}),"after-step":o.integer({description:"Only emit records with step_index > this value. Default: read from existing state file, else -1 (everything)."}),limit:o.integer({description:"Cap on the number of records to process this run."}),"dry-run":o.boolean({default:!1,description:"Classify records without POSTing. Reports counts only."}),"save-state":o.boolean({default:!0,allowNo:!0,description:"After backfill, persist the highest step_index so the live hook resumes from there."})};async run(){let{args:b,flags:e}=await this.parse(x),t=b.conversationId;d.loadEnvFile(),process.env.RULEMETRIC_API_URL||this.error("RULEMETRIC_API_URL is not set. Configure it via env or ~/.config/rulemetric/env.");let l=e.transcript??I(A,t,".system_generated","logs","transcript.jsonl");T(l)||this.error(`Transcript not found: ${l}`);let m=e.session??d.resolveRwSessionId(t);m||this.error(`No RuleMetric session resolved. Pass --session <uuid>, or run the Antigravity hook at least once so $TMPDIR/rulemetric/${t} is populated.`);let n=e["after-step"];n===void 0&&(n=d.load(t)?.lastStepIdx??-1);let i=v(l,n);e.limit!==void 0&&i.length>e.limit&&(i=i.slice(0,e.limit));let R=u(l);if(this.log(`Conversation ${t}: transcript max step = ${R}, after-step = ${n}, records to process = ${i.length}`),i.length===0){this.log("Nothing to backfill.");return}if(e["dry-run"]){let r=0,a=0,c={};for(let S of i){let f=(await import("../../lib/antigravity-transcript.js")).classifyRecord(S,0);f?(r++,c[f.eventType]=(c[f.eventType]??0)+1):a++}this.log(`Dry run: would emit ${r}, skip ${a}. By eventType: ${JSON.stringify(c)}`);return}let w=Date.now(),s=await y(i,m,t,n,{onProgress:p(({attempted:r,total:a})=>{(r%100===0||r===a)&&process.stderr.write(`  \u2026${r}/${a}
 `)},"onProgress")}),$=((Date.now()-w)/1e3).toFixed(1);this.log(`Backfill complete in ${$}s. attempted=${s.attempted} emitted=${s.emitted} skipped=${s.skipped} failed=${s.failed} highestStepIdx=${s.highestStepIdx}`),e["save-state"]?(d.save(t,{lastStepIdx:s.highestStepIdx}),this.log(`State saved \u2192 live hook will resume from step_index > ${s.highestStepIdx}.`)):this.log("State NOT saved (--no-save-state). Live hook will re-emit any new records from after-step.")}};export{g as default};

package/dist/commands/auth/create-key.js

@@ -1 +1 @@
-import{a as n}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{f as r}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{i as o,l as s}from"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import*as c from"node:path";import{Flags as l}from"@oclif/core";var t=class g extends n{static{a(this,"AuthCreateKey")}static description="Create a long-lived API key for CLI and proxy authentication";static examples=['<%= config.bin %> auth create-key --name "laptop"','<%= config.bin %> auth create-key --name "ci token" --no-save'];static flags={name:l.string({char:"n",description:"Name for this key",required:!0}),save:l.boolean({char:"s",description:"Save key to ~/.config/rulemetric/env for hooks/proxy",default:!0,allowNo:!0})};async run(){let{flags:i}=await this.parse(g),e=await r("/api/auth/api-keys",{name:i.name});this.log(""),this.log("API key created successfully:"),this.log(""),this.log(`  Name:   ${e.name}`),this.log(`  Key:    ${e.key}`),this.log(`  Prefix: ${e.keyPrefix}`),this.log(""),this.warn("Save this key now \u2014 it will not be shown again."),i.save&&(o(e.key,process.env.RULEMETRIC_API_URL),this.log(""),this.log(`Key saved to ${c.join(s(),"env")} (RULEMETRIC_API_KEY)`),this.log("Hooks and proxy will use this key automatically."))}};export{t as default};
+import{a as n}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{f as r}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{i as o,l as s}from"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import*as c from"node:path";import{Flags as l}from"@oclif/core";var t=class g extends n{static{a(this,"AuthCreateKey")}static description="Create a long-lived API key for CLI and proxy authentication";static examples=['<%= config.bin %> auth create-key --name "laptop"','<%= config.bin %> auth create-key --name "ci token" --no-save'];static flags={name:l.string({char:"n",description:"Name for this key",required:!0}),save:l.boolean({char:"s",description:"Save key to ~/.config/rulemetric/env for hooks/proxy",default:!0,allowNo:!0})};async run(){let{flags:i}=await this.parse(g),e=await r("/api/auth/api-keys",{name:i.name});this.log(""),this.log("API key created successfully:"),this.log(""),this.log(`  Name:   ${e.name}`),this.log(`  Key:    ${e.key}`),this.log(`  Prefix: ${e.keyPrefix}`),this.log(""),this.warn("Save this key now \u2014 it will not be shown again."),i.save&&(o(e.key,process.env.RULEMETRIC_API_URL),this.log(""),this.log(`Key saved to ${c.join(s(),"env")} (RULEMETRIC_API_KEY)`),this.log("Hooks and proxy will use this key automatically."))}};export{t as default};

package/dist/commands/auth/login.js

@@ -1,4 +1,4 @@
-import{a as f}from"../../chunk-QPVUGTDP.js";import{a as w}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import{c}from"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import{b as n}from"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{b as r,d as a}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{b as m,d,h as u,i as g}from"../../chunk-PZNWH4VQ.js";import{a as l}from"../../chunk-ZLLER3HP.js";import{Flags as p}from"@oclif/core";var h=class y extends w{static{l(this,"AuthLogin")}static description="Log in to RuleMetric";static examples=["<%= config.bin %> auth login","<%= config.bin %> auth login -e user@example.com"];static flags={email:p.string({char:"e",description:"Email address"}),password:p.string({char:"p",description:"Password (for non-interactive use)"}),"api-key":p.string({description:"Authenticate with an API key from https://rulemetric.com/connect (required for Google/GitHub sign-ins, which have no password)",exclusive:["password"]})};async run(){let{flags:s}=await this.parse(y);if(s["api-key"]){await this.loginWithApiKey(s["api-key"]);return}let e=s.email,t=s.password;if(!e||!t){process.stdin.isTTY||this.error("Missing credentials in a non-interactive shell. Pass --email/-e and --password/-p, or set RULEMETRIC_API_KEY (create a key at https://rulemetric.com/connect).");let{input:o,password:k}=await import("@inquirer/prompts");this.log("Signed up with Google or GitHub? You have no password \u2014 press Ctrl-C and run:"),this.log("  rulemetric auth login --api-key <key>   (create one at https://rulemetric.com/connect)"),this.log(""),e||(e=await o({message:"Email:"})),t||(t=await k({message:"Password:"}))}let i;try{i=await a("/auth/login",{method:"POST",body:{email:e,password:t},timeoutMs:2e4})}catch(o){throw o instanceof r&&o.statusCode===401&&this.error(`Invalid email or password.
+import{a as f}from"../../chunk-JCZOEIL7.js";import{a as w}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import{c}from"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import{b as n}from"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{b as r,d as a}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{b as m,d,h as u,i as g}from"../../chunk-PZNWH4VQ.js";import{a as l}from"../../chunk-ZLLER3HP.js";import{Flags as p}from"@oclif/core";var h=class y extends w{static{l(this,"AuthLogin")}static description="Log in to RuleMetric";static examples=["<%= config.bin %> auth login","<%= config.bin %> auth login -e user@example.com"];static flags={email:p.string({char:"e",description:"Email address"}),password:p.string({char:"p",description:"Password (for non-interactive use)"}),"api-key":p.string({description:"Authenticate with an API key from https://rulemetric.com/connect (required for Google/GitHub sign-ins, which have no password)",exclusive:["password"]})};async run(){let{flags:s}=await this.parse(y);if(s["api-key"]){await this.loginWithApiKey(s["api-key"]);return}let e=s.email,t=s.password;if(!e||!t){process.stdin.isTTY||this.error("Missing credentials in a non-interactive shell. Pass --email/-e and --password/-p, or set RULEMETRIC_API_KEY (create a key at https://rulemetric.com/connect).");let{input:o,password:k}=await import("@inquirer/prompts");this.log("Signed up with Google or GitHub? You have no password \u2014 press Ctrl-C and run:"),this.log("  rulemetric auth login --api-key <key>   (create one at https://rulemetric.com/connect)"),this.log(""),e||(e=await o({message:"Email:"})),t||(t=await k({message:"Password:"}))}let i;try{i=await a("/auth/login",{method:"POST",body:{email:e,password:t},timeoutMs:2e4})}catch(o){throw o instanceof r&&o.statusCode===401&&this.error(`Invalid email or password.
 
 If you signed up with Google or GitHub, you have no password \u2014 that is why this cannot work. Create an API key at https://rulemetric.com/connect, then run:
   rulemetric auth login --api-key <key>`),o}let v=i.expiresAt??new Date(Date.now()+3600*1e3).toISOString();d({accessToken:i.accessToken,refreshToken:i.refreshToken,expiresAt:v,email:i.email}),u(i.accessToken,process.env.RULEMETRIC_API_URL),await n(),await f(this.log.bind(this),this.warn.bind(this),this.config.version),c("auth_login",{},this.config.version),this.log(`Logged in as ${i.email}`)}async loginWithApiKey(s){let e=s.trim();e||this.error("Empty --api-key. Create one at https://rulemetric.com/connect"),process.env.RULEMETRIC_API_KEY=e;try{await a("/api/setup/status",{timeoutMs:2e4})}catch(t){throw delete process.env.RULEMETRIC_API_KEY,t instanceof r&&(t.statusCode===401||t.statusCode===403)&&this.error("That API key was rejected. Check you copied it whole (it is shown only once at creation) or mint a new one at https://rulemetric.com/connect"),t}g(e,process.env.RULEMETRIC_API_URL),m(),await n(),c("auth_login",{method:"api_key"},this.config.version),this.log(`Authenticated with API key ${e.slice(0,11)}...`),this.log("Verify any time with: rulemetric auth status")}};export{h as default};

package/dist/commands/auth/logout.js

@@ -1 +1 @@
-import{a as s}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{f as i,j as r}from"../../chunk-PZNWH4VQ.js";import{a as o}from"../../chunk-ZLLER3HP.js";var e=class t extends s{static{o(this,"AuthLogout")}static description="Log out of RuleMetric";static examples=["<%= config.bin %> auth logout"];async run(){await this.parse(t),i(),r(),this.log("Logged out successfully.")}};export{e as default};
+import{a as s}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{f as i,j as r}from"../../chunk-PZNWH4VQ.js";import{a as o}from"../../chunk-ZLLER3HP.js";var e=class t extends s{static{o(this,"AuthLogout")}static description="Log out of RuleMetric";static examples=["<%= config.bin %> auth logout"];async run(){await this.parse(t),i(),r(),this.log("Logged out successfully.")}};export{e as default};

package/dist/commands/auth/revoke-key.js

@@ -1,4 +1,4 @@
-import{a as k}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as n,i as c}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as s}from"../../chunk-ZLLER3HP.js";import{Args as y}from"@oclif/core";var a=class m extends k{static{s(this,"AuthRevokeKey")}static description="Revoke an API key (run with no argument to list active keys)";static examples=["<%= config.bin %> auth revoke-key","<%= config.bin %> auth revoke-key rm_abc12345",'<%= config.bin %> auth revoke-key "my-laptop (auto)"'];static args={id:y.string({description:"API key ID, prefix (rm_xxxxxxxx), or name"})};async run(){let{args:o}=await this.parse(m),r=(await n("/api/auth/api-keys")).filter(e=>!e.revokedAt);if(!o.id){if(r.length===0){this.log("No active API keys.");return}this.log(`Active API keys \u2014 pass an id, prefix, or name to revoke one:
+import{a as k}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as n,i as c}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as s}from"../../chunk-ZLLER3HP.js";import{Args as y}from"@oclif/core";var a=class m extends k{static{s(this,"AuthRevokeKey")}static description="Revoke an API key (run with no argument to list active keys)";static examples=["<%= config.bin %> auth revoke-key","<%= config.bin %> auth revoke-key rm_abc12345",'<%= config.bin %> auth revoke-key "my-laptop (auto)"'];static args={id:y.string({description:"API key ID, prefix (rm_xxxxxxxx), or name"})};async run(){let{args:o}=await this.parse(m),r=(await n("/api/auth/api-keys")).filter(e=>!e.revokedAt);if(!o.id){if(r.length===0){this.log("No active API keys.");return}this.log(`Active API keys \u2014 pass an id, prefix, or name to revoke one:
 `);for(let e of r)this.log(`  ${e.id}  ${e.keyPrefix}...  ${e.name}`);return}let i=o.id,t=r.filter(e=>e.id===i||e.name===i||i.startsWith("rm_")&&i.startsWith(e.keyPrefix));t.length===0&&this.error(`No active API key matches "${i}". Run \`rulemetric auth revoke-key\` with no argument to list keys.`),t.length>1&&this.error(`"${i}" matches ${t.length} keys \u2014 use the exact key ID:
 `+t.map(e=>`  ${e.id}  ${e.keyPrefix}...  ${e.name}`).join(`
 `)),await c(`/api/auth/api-keys/${t[0].id}`),this.log(`API key ${t[0].keyPrefix}... ("${t[0].name}") revoked.`)}};export{a as default};

package/dist/commands/auth/signup.js

@@ -1 +1 @@
-import{a as g}from"../../chunk-QPVUGTDP.js";import{a as h}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import{c as d}from"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import{b as l}from"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{b as m,d as p}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{d as n,h as c}from"../../chunk-PZNWH4VQ.js";import{a as o}from"../../chunk-ZLLER3HP.js";import{Flags as u}from"@oclif/core";var t=class f extends h{static{o(this,"AuthSignup")}static description="Create a RuleMetric account from the CLI";static examples=["<%= config.bin %> auth signup","<%= config.bin %> auth signup -e user@example.com"];static flags={email:u.string({char:"e",description:"Email address"}),password:u.string({char:"p",description:"Password (min 8 chars; for non-interactive use)"})};async run(){let{flags:a}=await this.parse(f),i=a.email,r=a.password;if(!i||!r){process.stdin.isTTY||this.error("Missing credentials in a non-interactive shell. Pass --email/-e and --password/-p.");let{input:s,password:v}=await import("@inquirer/prompts");i||(i=await s({message:"Email:"})),r||(r=await v({message:"Password (min 8 chars):"}))}let e;try{e=await p("/auth/signup",{method:"POST",body:{email:i,password:r},timeoutMs:2e4})}catch(s){throw s instanceof m&&s.statusCode===409&&this.error(`An account with ${i} already exists. Run: rulemetric auth login -e ${i}`),s}if(e.confirmationRequired||!e.accessToken||!e.refreshToken){this.log(`Account created for ${e.email}.`),this.log("Check your email and click the confirmation link, then run:"),this.log(`  rulemetric auth login -e ${e.email}`);return}let w=e.expiresAt??new Date(Date.now()+3600*1e3).toISOString();n({accessToken:e.accessToken,refreshToken:e.refreshToken,expiresAt:w,email:e.email}),c(e.accessToken,process.env.RULEMETRIC_API_URL),await l(),await g(this.log.bind(this),this.warn.bind(this),this.config.version),d("auth_signup",{},this.config.version),this.log(`Signed up and logged in as ${e.email}`)}};export{t as default};
+import{a as g}from"../../chunk-JCZOEIL7.js";import{a as h}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import{c as d}from"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import{b as l}from"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{b as m,d as p}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{d as n,h as c}from"../../chunk-PZNWH4VQ.js";import{a as o}from"../../chunk-ZLLER3HP.js";import{Flags as u}from"@oclif/core";var t=class f extends h{static{o(this,"AuthSignup")}static description="Create a RuleMetric account from the CLI";static examples=["<%= config.bin %> auth signup","<%= config.bin %> auth signup -e user@example.com"];static flags={email:u.string({char:"e",description:"Email address"}),password:u.string({char:"p",description:"Password (min 8 chars; for non-interactive use)"})};async run(){let{flags:a}=await this.parse(f),i=a.email,r=a.password;if(!i||!r){process.stdin.isTTY||this.error("Missing credentials in a non-interactive shell. Pass --email/-e and --password/-p.");let{input:s,password:v}=await import("@inquirer/prompts");i||(i=await s({message:"Email:"})),r||(r=await v({message:"Password (min 8 chars):"}))}let e;try{e=await p("/auth/signup",{method:"POST",body:{email:i,password:r},timeoutMs:2e4})}catch(s){throw s instanceof m&&s.statusCode===409&&this.error(`An account with ${i} already exists. Run: rulemetric auth login -e ${i}`),s}if(e.confirmationRequired||!e.accessToken||!e.refreshToken){this.log(`Account created for ${e.email}.`),this.log("Check your email and click the confirmation link, then run:"),this.log(`  rulemetric auth login -e ${e.email}`);return}let w=e.expiresAt??new Date(Date.now()+3600*1e3).toISOString();n({accessToken:e.accessToken,refreshToken:e.refreshToken,expiresAt:w,email:e.email}),c(e.accessToken,process.env.RULEMETRIC_API_URL),await l(),await g(this.log.bind(this),this.warn.bind(this),this.config.version),d("auth_signup",{},this.config.version),this.log(`Signed up and logged in as ${e.email}`)}};export{t as default};

package/dist/commands/auth/status.js

@@ -1 +1 @@
-import{a as c}from"../../chunk-W2MV25SB.js";import{a as g}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{a,e as l,l as t}from"../../chunk-PZNWH4VQ.js";import{a as r}from"../../chunk-ZLLER3HP.js";import*as u from"node:path";var o=class h extends g{static{r(this,"AuthStatus")}static description="Show current authentication status";static examples=["<%= config.bin %> auth status"];async run(){await this.parse(h);let e=l(),i=process.env.RULEMETRIC_API_KEY,s=a().apiKey;if(i||s){let m=i||s,p=i?"RULEMETRIC_API_KEY env var":u.join(t(),"env");this.log(`Authenticated via API key ${m.slice(0,11)}... (${p})`),e&&this.log(`Logged in as: ${e.email}`),this.log(`Config: ${t()}`);return}if(!e){this.log("Not logged in. Run `rulemetric auth login` to authenticate.");return}let n=new Date(e.expiresAt).getTime();if(Number.isNaN(n)||Date.now()>=n){this.log("Session expired. Run `rulemetric auth login` to re-authenticate.");return}this.log(`Logged in as: ${e.email}`),this.log(`Session expires: ${c(e.expiresAt)}`),this.log(`Config: ${t()}`)}};export{o as default};
+import{a as c}from"../../chunk-W2MV25SB.js";import{a as g}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{a,e as l,l as t}from"../../chunk-PZNWH4VQ.js";import{a as r}from"../../chunk-ZLLER3HP.js";import*as u from"node:path";var o=class h extends g{static{r(this,"AuthStatus")}static description="Show current authentication status";static examples=["<%= config.bin %> auth status"];async run(){await this.parse(h);let e=l(),i=process.env.RULEMETRIC_API_KEY,s=a().apiKey;if(i||s){let m=i||s,p=i?"RULEMETRIC_API_KEY env var":u.join(t(),"env");this.log(`Authenticated via API key ${m.slice(0,11)}... (${p})`),e&&this.log(`Logged in as: ${e.email}`),this.log(`Config: ${t()}`);return}if(!e){this.log("Not logged in. Run `rulemetric auth login` to authenticate.");return}let n=new Date(e.expiresAt).getTime();if(Number.isNaN(n)||Date.now()>=n){this.log("Session expired. Run `rulemetric auth login` to re-authenticate.");return}this.log(`Logged in as: ${e.email}`),this.log(`Session expires: ${c(e.expiresAt)}`),this.log(`Config: ${t()}`)}};export{o as default};

package/dist/commands/capture/scope.js

@@ -1,2 +1,2 @@
-import{a as f,b as k,c as S,d as x,e as R,f as n,g as y}from"../../chunk-VALRPZXH.js";import{b as C,c as w,g as v,h as b}from"../../chunk-QI3FJKGK.js";import{a as u}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{l as m}from"../../chunk-PZNWH4VQ.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{existsSync as O}from"node:fs";import{Flags as c}from"@oclif/core";var g=class E extends u{static{h(this,"CaptureScope")}static description="Show or set which projects RuleMetric captures \u2014 the Codex sweep and hook-driven sessions each have their own scope";static examples=["<%= config.bin %> <%= command.id %>","<%= config.bin %> <%= command.id %> --add ~/work/client-a --add ~/work/client-b","<%= config.bin %> <%= command.id %> --tool hooks --add ~/work/client-a","<%= config.bin %> <%= command.id %> --tool hooks --off","<%= config.bin %> <%= command.id %> --remove ~/work/client-a","<%= config.bin %> <%= command.id %> --off","<%= config.bin %> <%= command.id %> --all"];static flags={tool:c.string({description:'Which capture path to edit: "codex" (the worker transcript sweep, the default) or "hooks" (Claude Code / Cursor / Copilot / Antigravity hook sessions).',options:["codex","hooks"],default:"codex"}),add:c.string({description:"Capture sessions under this path (repeatable). Implies an allowlist.",multiple:!0}),remove:c.string({description:"Stop capturing under this path (repeatable).",multiple:!0}),off:c.boolean({description:"Turn this capture path off entirely.",exclusive:["all"]}),all:c.boolean({description:"Capture every project (the default).",exclusive:["off"]}),clear:c.boolean({description:"Remove this scope, restoring the default."})};async run(){let{flags:o}=await this.parse(E),s=m(),d=f(s);if(!((o.add?.length??0)>0||(o.remove?.length??0)>0||o.off||o.all||o.clear)){await this.show(s,d);return}for(let t of[...o.add??[],...o.remove??[]])R(t)||this.error(`Not an absolute path: ${t}
+import{a as f,b as k,c as S,d as x,e as R,f as n,g as y}from"../../chunk-VALRPZXH.js";import{b as C,c as w,g as v,h as b}from"../../chunk-QI3FJKGK.js";import{a as u}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{l as m}from"../../chunk-PZNWH4VQ.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{existsSync as O}from"node:fs";import{Flags as c}from"@oclif/core";var g=class E extends u{static{h(this,"CaptureScope")}static description="Show or set which projects RuleMetric captures \u2014 the Codex sweep and hook-driven sessions each have their own scope";static examples=["<%= config.bin %> <%= command.id %>","<%= config.bin %> <%= command.id %> --add ~/work/client-a --add ~/work/client-b","<%= config.bin %> <%= command.id %> --tool hooks --add ~/work/client-a","<%= config.bin %> <%= command.id %> --tool hooks --off","<%= config.bin %> <%= command.id %> --remove ~/work/client-a","<%= config.bin %> <%= command.id %> --off","<%= config.bin %> <%= command.id %> --all"];static flags={tool:c.string({description:'Which capture path to edit: "codex" (the worker transcript sweep, the default) or "hooks" (Claude Code / Cursor / Copilot / Antigravity hook sessions).',options:["codex","hooks"],default:"codex"}),add:c.string({description:"Capture sessions under this path (repeatable). Implies an allowlist.",multiple:!0}),remove:c.string({description:"Stop capturing under this path (repeatable).",multiple:!0}),off:c.boolean({description:"Turn this capture path off entirely.",exclusive:["all"]}),all:c.boolean({description:"Capture every project (the default).",exclusive:["off"]}),clear:c.boolean({description:"Remove this scope, restoring the default."})};async run(){let{flags:o}=await this.parse(E),s=m(),d=f(s);if(!((o.add?.length??0)>0||(o.remove?.length??0)>0||o.off||o.all||o.clear)){await this.show(s,d);return}for(let t of[...o.add??[],...o.remove??[]])R(t)||this.error(`Not an absolute path: ${t}
 Use an absolute path or ~/\u2026 \u2014 relative paths cannot be matched against a session's cwd.`);for(let t of o.add??[]){let a=x(t);O(a)||this.warn(`${a} does not exist yet \u2014 added anyway, but nothing will match until it does.`)}let e;try{e=k(s)}catch(t){o.clear||this.error(t instanceof Error?t.message:String(t));let a=y(s);this.warn(`${f(s)} was not valid JSON \u2014 saved it to ${a} and starting fresh.`),e={}}let i=o.tool??"codex",r=void 0,l=!1;if(o.clear&&({config:e}=n(e,{op:"clear"},r,i)),o.off&&({config:e}=n(e,{op:"mode",mode:"off"},r,i)),o.all&&({config:e}=n(e,{op:"mode",mode:"all"},r,i)),o.remove?.length){let t=n(e,{op:"remove",roots:o.remove},r,i);e=t.config,l=t.emptiedToOff}o.add?.length&&({config:e}=n(e,{op:"add",roots:o.add},r,i)),S(s,e),l&&(this.log(`Removed the last root \u2014 ${i==="hooks"?"hook":"Codex"} capture is now OFF rather than silently capturing everything.`),this.log("Use --all to go back to capturing every project.")),await this.show(s,d),this.log(""),this.log(i==="hooks"?"Takes effect on the next hook fire. No restart needed.":"Takes effect on the next sweep (within ~30s). No restart needed.")}async show(o,s){let d=await C({configDir:o}),p=await w({configDir:o}),e=h((i,r,l,t)=>{if(this.log(`${i}: ${l}`),r.roots.length)for(let a of r.roots)this.log(`  \u2022 ${a}`);r.error&&(this.log("  This scope is FAILING CLOSED \u2014 nothing is being captured on this path."),this.log(`  Fix the file, or run \`rulemetric capture scope ${i==="Hook capture"?"--tool hooks ":""}--clear\` to start over.`)),r.source==="env"&&this.log(`  Set by environment variable, which outranks the config file (${t}).`)},"print");e("Codex capture",d,v(d),"RULEMETRIC_CODEX_CAPTURE / RULEMETRIC_CODEX_ROOTS"),e("Hook capture",p,b(p),"RULEMETRIC_HOOKS_CAPTURE / RULEMETRIC_HOOKS_ROOTS"),this.log(`Config: ${s}`)}};export{g as default};

package/dist/commands/catalog/index.js

@@ -1,2 +1,2 @@
-import{a as g,b as d,c as p}from"../../chunk-W2MV25SB.js";import{a as c}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import{Flags as i}from"@oclif/core";var o=class m extends c{static{a(this,"Catalog")}static description="Browse the org instruction catalog";static flags={badge:i.string({description:"Filter by badge",options:["recommended","required"]}),scope:i.string({description:"Filter by scope",options:["org","team"]}),search:i.string({description:"Search by name or description"})};async run(){let{flags:e}=await this.parse(m);this.requireAuth();let s=new URLSearchParams;e.badge&&s.set("badge",e.badge),e.scope&&s.set("scope",e.scope),e.search&&s.set("search",e.search);let r=await n(`/api/catalog?${s.toString()}`);if(r.length===0){this.log("No instructions found in catalog.");return}for(let t of r){let l=t.badge?` [${t.badge}]`:"",f=t.tags?.length?` (${t.tags.join(", ")})`:"";this.log(`  ${d(t.name,40)} \u2014 ${p(t.scope)}${l} v${t.version}${f}  ${g(t.updatedAt)}`)}this.log(`
+import{a as g,b as d,c as p}from"../../chunk-W2MV25SB.js";import{a as c}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import{Flags as i}from"@oclif/core";var o=class m extends c{static{a(this,"Catalog")}static description="Browse the org instruction catalog";static flags={badge:i.string({description:"Filter by badge",options:["recommended","required"]}),scope:i.string({description:"Filter by scope",options:["org","team"]}),search:i.string({description:"Search by name or description"})};async run(){let{flags:e}=await this.parse(m);this.requireAuth();let s=new URLSearchParams;e.badge&&s.set("badge",e.badge),e.scope&&s.set("scope",e.scope),e.search&&s.set("search",e.search);let r=await n(`/api/catalog?${s.toString()}`);if(r.length===0){this.log("No instructions found in catalog.");return}for(let t of r){let l=t.badge?` [${t.badge}]`:"",f=t.tags?.length?` (${t.tags.join(", ")})`:"";this.log(`  ${d(t.name,40)} \u2014 ${p(t.scope)}${l} v${t.version}${f}  ${g(t.updatedAt)}`)}this.log(`
 ${r.length} instruction(s) found.`)}};export{o as default};

package/dist/commands/catalog/recommend.js

@@ -1,3 +1,3 @@
-import{a as i}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{f as n}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as t}from"../../chunk-ZLLER3HP.js";import{Args as c}from"@oclif/core";var o=class s extends i{static{t(this,"CatalogRecommend")}static description="Get AI-powered instruction recommendations for a project";static args={projectId:c.string({description:"ID of the project",required:!0})};async run(){let{args:a}=await this.parse(s);this.requireAuth(),this.log(`Analyzing project and generating recommendations...
+import{a as i}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{f as n}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as t}from"../../chunk-ZLLER3HP.js";import{Args as c}from"@oclif/core";var o=class s extends i{static{t(this,"CatalogRecommend")}static description="Get AI-powered instruction recommendations for a project";static args={projectId:c.string({description:"ID of the project",required:!0})};async run(){let{args:a}=await this.parse(s);this.requireAuth(),this.log(`Analyzing project and generating recommendations...
 `);let r=await n(`/api/projects/${a.projectId}/recommendations`,{});if(r.error&&this.warn(`Note: ${r.error}`),r.recommendations.length===0){this.log("No recommendations found.");return}for(let e of r.recommendations)this.log(`  ${(e.score*100).toFixed(0)}% \u2014 ${e.instructionId}`),this.log(`       ${e.reason}
 `)}};export{o as default};

package/dist/commands/crons/list.js

@@ -1 +1 @@
-import{a as o}from"../../chunk-4VN5PG2S.js";import{a as n}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as i}from"../../chunk-ZLLER3HP.js";import{printTable as t}from"@oclif/table";var a=class s extends n{static{i(this,"CronsList")}static description="List manually-runnable maintenance tasks";static examples=["<%= config.bin %> crons list"];async run(){await this.parse(s),t({data:o.map(e=>({Name:e.name,"Scheduled by":e.scheduledBy,Description:e.description})),columns:["Name","Scheduled by","Description"]}),this.log(""),this.log("Scheduled crons run in Supabase pg_cron \u2014 audit them with:"),this.log(`  psql "$DATABASE_URL" -c 'select jobname, schedule, active from cron.job;'`),this.log("Run any task above on demand with: rulemetric crons run <name>")}};export{a as default};
+import{a as o}from"../../chunk-4VN5PG2S.js";import{a as n}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as i}from"../../chunk-ZLLER3HP.js";import{printTable as t}from"@oclif/table";var a=class s extends n{static{i(this,"CronsList")}static description="List manually-runnable maintenance tasks";static examples=["<%= config.bin %> crons list"];async run(){await this.parse(s),t({data:o.map(e=>({Name:e.name,"Scheduled by":e.scheduledBy,Description:e.description})),columns:["Name","Scheduled by","Description"]}),this.log(""),this.log("Scheduled crons run in Supabase pg_cron \u2014 audit them with:"),this.log(`  psql "$DATABASE_URL" -c 'select jobname, schedule, active from cron.job;'`),this.log("Run any task above on demand with: rulemetric crons run <name>")}};export{a as default};

package/dist/commands/crons/run.js

@@ -1,2 +1,2 @@
-import{a as $}from"../../chunk-4VN5PG2S.js";import{a as k}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as b}from"../../chunk-GIXHG5FK.js";import{A as f,B as g,z as h}from"../../chunk-X6CF5KTW.js";import{f as p}from"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as d}from"../../chunk-ZLLER3HP.js";import{Args as _,Flags as y}from"@oclif/core";var l=class w extends k{static{d(this,"CronsRun")}static description="Run a maintenance task synchronously, right now";static examples=["<%= config.bin %> crons run cron_check_changelog","<%= config.bin %> crons run cron_suggest_instructions --quiet"];static args={name:_.string({description:"Task name (run `rulemetric crons list` to see all)",required:!0})};static flags={"dry-run":y.boolean({description:"For Harbor, print the saved engine/model and supported tasks without running them"}),quiet:y.boolean({char:"q",description:"Suppress per-line log output; print summary only"})};async run(){let{args:s,flags:a}=await this.parse(w);this.requireAuth();let o=$.find(e=>e.name===s.name);o||this.error(`Unknown task: ${s.name}
+import{a as $}from"../../chunk-4VN5PG2S.js";import{a as k}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as b}from"../../chunk-5VWVQCNR.js";import{A as f,B as g,z as h}from"../../chunk-MHLHLJ5T.js";import{f as p}from"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as d}from"../../chunk-ZLLER3HP.js";import{Args as _,Flags as y}from"@oclif/core";var l=class w extends k{static{d(this,"CronsRun")}static description="Run a maintenance task synchronously, right now";static examples=["<%= config.bin %> crons run cron_check_changelog","<%= config.bin %> crons run cron_suggest_instructions --quiet"];static args={name:_.string({description:"Task name (run `rulemetric crons list` to see all)",required:!0})};static flags={"dry-run":y.boolean({description:"For Harbor, print the saved engine/model and supported tasks without running them"}),quiet:y.boolean({char:"q",description:"Suppress per-line log output; print summary only"})};async run(){let{args:s,flags:a}=await this.parse(w);this.requireAuth();let o=$.find(e=>e.name===s.name);o||this.error(`Unknown task: ${s.name}
 Run \`rulemetric crons list\` to see available tasks.`);let t=null;if(s.name==="harbor_ab"){let e=await b("/api/settings/measurement");t=e.execution.harbor?p.parse(e.execution.harbor):null;for(let r of h("proven")){let m=f(r,t);if(m){this.log(`Unsupported: ${m}`);continue}let i=g(r,"manual",t);this.log(`${r.task}: ${t?.engine??"claude"} / ${i.model}, ${i.class}, ${i.nTrials} trials per arm`)}if(a["dry-run"])return}else a["dry-run"]&&this.error("--dry-run is currently supported only for harbor_ab");let u;try{({runManualTask:u}=await import("../../lib/manual-tasks.js"))}catch(e){let r=e?.code;throw(r==="ERR_MODULE_NOT_FOUND"||r==="MODULE_NOT_FOUND")&&this.error("`crons run` needs the worker runtime, which isn't available in this install.\nReinstall the CLI and verify that its runtime dependencies are present. A database credential is not required for end-user maintenance tasks."),e}this.log(`\u25B6 Running ${o.name} \u2026`);let v=Date.now(),n=await u(s.name,{measurement:t});if(!a.quiet)for(let e of n.logs){let r=e.level==="error"||e.level==="warn"?`[${e.level}]`:"";this.log(`  ${r} ${e.message}`)}let c=((n.durationMs??Date.now()-v)/1e3).toFixed(1);n.ok?this.log(`\u2713 ${o.name} completed in ${c}s (${n.logs.length} log line${n.logs.length===1?"":"s"})`):this.error(`\u2717 ${o.name} failed after ${c}s: ${n.error}`)}};export{l as default};

package/dist/commands/dashboard.js

@@ -1 +1 @@
-import{a as o}from"../chunk-WUCACPGC.js";import"../chunk-IZTIYR5Q.js";import"../chunk-WL5ZWQAS.js";import"../chunk-KEVDLLHT.js";import"../chunk-DX6IPDED.js";import"../chunk-3AV3GTMP.js";import"../chunk-GIXHG5FK.js";import"../chunk-X6CF5KTW.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import{a as i}from"../chunk-ZLLER3HP.js";import{Flags as r}from"@oclif/core";var a=class s extends o{static{i(this,"Dashboard")}static description="Interactive TUI dashboard showing session analytics, outcomes, friction, and insights";static examples=["<%= config.bin %> dashboard","<%= config.bin %> dashboard --period 30days","<%= config.bin %> dashboard --launcher"];static flags={period:r.string({char:"p",default:"week",options:["today","week","30days","month","all"],description:"Initial time period"}),launcher:r.boolean({char:"l",default:!1,description:"Open the launcher overlay immediately"})};async run(){let{flags:e}=await this.parse(s),{renderDashboard:t}=await import("../dashboard/Dashboard.js");await t(e.period,{openLauncher:e.launcher})}};export{a as default};
+import{a as o}from"../chunk-TQFHRD5B.js";import"../chunk-IZTIYR5Q.js";import"../chunk-5LTJNYCP.js";import"../chunk-3EZZGA72.js";import"../chunk-XAGNGQL3.js";import"../chunk-3AV3GTMP.js";import"../chunk-5VWVQCNR.js";import"../chunk-MHLHLJ5T.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import{a as i}from"../chunk-ZLLER3HP.js";import{Flags as r}from"@oclif/core";var a=class s extends o{static{i(this,"Dashboard")}static description="Interactive TUI dashboard showing session analytics, outcomes, friction, and insights";static examples=["<%= config.bin %> dashboard","<%= config.bin %> dashboard --period 30days","<%= config.bin %> dashboard --launcher"];static flags={period:r.string({char:"p",default:"week",options:["today","week","30days","month","all"],description:"Initial time period"}),launcher:r.boolean({char:"l",default:!1,description:"Open the launcher overlay immediately"})};async run(){let{flags:e}=await this.parse(s),{renderDashboard:t}=await import("../dashboard/Dashboard.js");await t(e.period,{openLauncher:e.launcher})}};export{a as default};

package/dist/commands/diagnostics.js

@@ -1 +1 @@
-import{c as q,d as K}from"../chunk-TYFEWLYZ.js";import{b as L,c as W}from"../chunk-V5U3E4BZ.js";import{a as J}from"../chunk-2TJ7Q2OU.js";import{e as O,f as _,g as M}from"../chunk-QWCEZEDO.js";import"../chunk-KKDHD4XA.js";import"../chunk-SZ62CP55.js";import{d as oe,e as ne}from"../chunk-KGU2IW54.js";import"../chunk-M7QSC64D.js";import"../chunk-ZKPURXUS.js";import"../chunk-ISEUEYSC.js";import{i as z,j as Q,k as Z,m as ee,n as te}from"../chunk-GYKYJJNF.js";import"../chunk-G7V4ZZZ3.js";import"../chunk-LJE5J3EW.js";import"../chunk-AHLVY52I.js";import"../chunk-YXMAGC53.js";import"../chunk-T7HZZ4DJ.js";import"../chunk-DHIEGG4T.js";import"../chunk-PPAHIVG5.js";import"../chunk-5MXZJWQS.js";import"../chunk-CT7FWSIU.js";import"../chunk-N7ODX6PO.js";import{a as V}from"../chunk-OZQNSELV.js";import"../chunk-HFGO5762.js";import"../chunk-VS5ROGDP.js";import"../chunk-FADINV4H.js";import"../chunk-SWSOEYC7.js";import"../chunk-U2NT5OSQ.js";import"../chunk-KGURGIZ2.js";import"../chunk-GYNZ3OZF.js";import"../chunk-UPFH72PF.js";import"../chunk-FG343JM3.js";import"../chunk-CDY7TQFZ.js";import"../chunk-KFNFLMUR.js";import"../chunk-4GOXEEXZ.js";import"../chunk-JP6ULOEX.js";import"../chunk-JNLDISBA.js";import{d as F}from"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import{a as X,b as D,c as Y,g as j,h as G}from"../chunk-QI3FJKGK.js";import{a as B}from"../chunk-WUCACPGC.js";import"../chunk-IZTIYR5Q.js";import"../chunk-WL5ZWQAS.js";import"../chunk-KEVDLLHT.js";import"../chunk-DX6IPDED.js";import"../chunk-3AV3GTMP.js";import{a as U,e as y}from"../chunk-GIXHG5FK.js";import"../chunk-X6CF5KTW.js";import"../chunk-XQYAVSUQ.js";import{a as N,g as v,l as x}from"../chunk-PZNWH4VQ.js";import{a as S}from"../chunk-ZLLER3HP.js";import{spawnSync as le}from"node:child_process";import{existsSync as ce,readFileSync as ue}from"node:fs";import{join as se}from"node:path";var I=class h extends B{static{S(this,"Diagnostics")}static description='Diagnose capture / proxy / connectivity problems (e.g. a client-side "502 Bad Gateway")';static examples=["<%= config.bin %> diagnostics"];baseUrl(){return process.env.RULEMETRIC_API_URL??N().apiUrl??U}async checkDirect(){let r=process.env.HTTPS_PROXY,i=process.env.HTTP_PROXY;delete process.env.HTTPS_PROXY,delete process.env.HTTP_PROXY;let s=new AbortController,l=setTimeout(()=>s.abort(),8e3);try{return(await globalThis.fetch(`${this.baseUrl()}/health`,{signal:s.signal})).ok}catch{return!1}finally{clearTimeout(l),r!==void 0&&(process.env.HTTPS_PROXY=r),i!==void 0&&(process.env.HTTP_PROXY=i)}}configuredProxy(){if(process.env.HTTPS_PROXY)return process.env.HTTPS_PROXY;for(let r of["settings.local.json","settings.json"]){let i=se(process.cwd(),".claude",r);if(ce(i))try{let s=JSON.parse(ue(i,"utf8"));if(s.env?.HTTPS_PROXY)return s.env.HTTPS_PROXY}catch{}}return null}checkThroughProxy(r){let i=le("curl",["-sS","-x",r,"--max-time","8","-o","/dev/null","-w","%{http_code}",`${this.baseUrl()}/health`],{encoding:"utf8",stdio:["ignore","pipe","ignore"]});return i.error||typeof i.stdout!="string"?null:i.stdout.trim()==="200"}async run(){await this.parse(h);let r=process.version,i=this.configuredProxy(),s=i!=null,[l,m,d]=await Promise.all([this.checkDirect(),M(O),M(_)]),p=s&&i?this.checkThroughProxy(i):null,t=S((e,c,T)=>{let u=c===null?"\u2013":c?"[OK]":"[!!]";this.log(`  ${u} ${e.padEnd(22)} ${T}`)},"line");this.log("RuleMetric diagnostics"),this.log(""),t("Node",!L(r),`${r}${L(r)?" (< 20.11 \u2014 breaks the bundled proxy)":""}`),t("API (direct)",l,`${this.baseUrl()}/health ${l?"reachable":"UNREACHABLE"}`),t("Authenticated",v(),v()?"yes":"no \u2014 run `rulemetric auth login`"),t("Proxy configured",s?!0:null,s?i:"no HTTPS_PROXY set"),s&&(t("Gateway :"+O,m,m?"listening":"NOT listening"),t("mitmproxy :"+_,d,d?"listening":"not listening (gateway should route direct)"),t("API (via proxy)",p,p===null?"inconclusive (curl unavailable)":p?"reachable":"FAILED \u2014 502/blocked through your proxy"));let a=F();t("claude CLI",a.claudeBin!=null,a.claudeBin??"not found \u2014 LLM features (grading, distill) will no-op"),t("codex CLI",a.codexBin!=null?!0:null,a.codexBin?`${a.codexBin}${a.codexConfiguredModel?` (model ${a.codexConfiguredModel})`:""}`:"not found \u2014 optional; enables the cross-vendor reviewer (evals validate-judge)");let o=null;if(v()&&l)try{o=await y("/api/workers/health")}catch{o=null}if(o){let e=o.worker;t("Worker",!!e.active,e.present?e.active?`online (last heartbeat ${e.ageSeconds??"?"}s ago${e.clientVersion?`, v${e.clientVersion}`:""})`:`NOT RUNNING \u2014 last heartbeat ${A(e.ageSeconds)} ago`:"never connected \u2014 no worker has ever reported in");let c=o.worker.engines;if(c!=null){let u=[c.claude&&"claude",c.codex&&"codex"].filter(Boolean);t("LLM engine",u.length>0,u.length>0?`${u.join(" + ")} available to the worker`:"NONE \u2014 the worker found no `claude` and no `codex`. LLM jobs will complete with empty output rather than failing.")}let T=o.pending.reduce((u,E)=>u+E.count,0);if(T>0){let u=o.verdict.stuck.reduce(($,k)=>$+k.count,0),E=o.verdict.unclaimable.reduce(($,k)=>$+k.count,0),R=(o.verdict.engineBlocked??[]).reduce(($,k)=>$+k.count,0);t("Job queue",u===0&&R===0,`${T} pending`+(u>0?` \u2014 ${u} waiting on this worker`:"")+(R>0?` \u2014 ${R} need a local LLM engine`:"")+(E>0?` \u2014 ${E} need a worker with DATABASE_URL`:""))}else t("Job queue",!0,"empty")}else v()&&l&&t("Worker",null,"could not read /workers/health (older API?)");let n=o?.worker.machine;if(n){let e=n.suspends+n.stalls,c=n.lastPause;t("Worker pauses",e===0?!0:null,e===0?`none since ${n.since} \u2014 no sleep or stall seen by the worker`:`${n.suspends} suspend${n.suspends===1?"":"s"}`+(n.stalls>0?` + ${n.stalls} event-loop stall${n.stalls===1?"":"s"}`:"")+` since ${n.since}, ${A(Math.round(n.pausedMs/1e3))} paused total`+(c?`; last: ${c.kind} of ${A(Math.round(c.wallGapMs/1e3))} ending ${c.detectedAt}`:"")+" \u2014 jobs in flight during a suspend lose their lease and are redone on wake")}let f=await q();if(f){t("Machine sleep (pmset)",f.sleeps===0?!0:null,K(f));for(let e of f.recent.slice(-3)){let c=e.seconds!=null?` (${A(e.seconds)})`:"";this.log(`        \u21B3 ${e.at.toISOString()} ${e.kind}${c}: ${e.reason.slice(0,80)}`)}}let C=V(process.cwd()),g=await D({configDir:x()}),re=await Y({configDir:x()}),w=` \u2014 ${G(re)}`,b=J(process.cwd()),H=[b.user&&`user hooks (${b.user})`,b.workspace&&`workspace hooks (${b.workspace})`].filter(Boolean),ie={claude_code:`session hooks (+ proxy for instruction linking)${w}`,codex:`worker transcript sync of ~/.codex/sessions \u2014 ${j(g)}`,cursor:`user hooks (~/.cursor/hooks.json)${w}`,copilot_agent:`workspace hooks${w}`,antigravity:H.length>0?`${H.join(" + ")}${w}`:"NOT CAPTURED \u2014 no rulemetric block in ~/.gemini/config/hooks.json or .agents/hooks.json; run `rulemetric hooks install` (user hooks are on by default)"};this.log(""),this.log("  Capture coverage (agents found on this machine):");for(let e of C)this.log(`    \u2022 ${e.padEnd(15)} ${ie[e]??"captured"}`);if(C.includes("codex"))if(g.error)this.log(`    ! ${g.error}`);else if(g.mode==="allowlist")for(let e of g.roots)this.log(`        \u21B3 ${e}`);else g.mode==="all"&&this.log(`        \u21B3 scope it with ${se(x(),X)} ({"codex":{"roots":["/path/to/repo"]}}) or RULEMETRIC_CODEX_CAPTURE=off`);o&&!o.worker.active&&C.includes("codex")&&g.mode!=="off"&&this.log("    ! Codex capture runs INSIDE the worker \u2014 it is stopped while the worker is down."),v()&&l&&await this.reportEvolutionSpend();let P=W({nodeVersion:r,directApiOk:l,httpsProxySet:s,gatewayAlive:m,mitmproxyAlive:d,proxyPathOk:p});this.log("");let ae=P.severity==="ok"?"[OK]":P.severity==="warn"?"[!!]":"[XX]";this.log(`${ae} ${P.headline}`);for(let e of P.hints)this.log(`   \u2022 ${e}`);if(o&&o.verdict.severity!=="ok"){let e=o.verdict.severity==="warn"?"[!!]":"[XX]";this.log(""),this.log(`${e} ${o.verdict.headline}`);for(let c of o.verdict.hints)this.log(`   \u2022 ${c}`)}}async reportEvolutionSpend(){let r;try{r=await y("/api/eval-targets?type=instruction&limit=200")}catch{return}let i=r.filter(s=>{let l=s.metadata??{};return l.autoEvolveEnabled===!0||l.autoEvolveEnabled==="true"});if(i.length!==0){this.log(""),this.log("  Nightly instruction evolution (cron_instruction_evolution, 07:30 UTC):");for(let s of i){let l=s.metadata??{},m=Q(l),d=z(l);if(m){this.log(`    \u2022 ${s.name} \u2014 PAUSED (metadata.autoEvolvePaused). Opt-in preserved.`),this.log("        \u21B3 resume: set autoEvolvePaused=false");continue}let p=null;try{p=(await y(`/api/eval-targets/${s.id}/evals`)).filter(a=>!a.retiredAt).length}catch{}if(this.log(`    \u2022 ${s.name}`),p!=null){let t=ne(p,d.minPairs,oe(d,p));if(!t.feasible)this.log(`        \u21B3 cannot reach a verdict \u2014 ${t.reason}`);else{let a=Z({liveCases:p,passes:t.passes});this.log(`        \u21B3 each night: ${te(a)}`)}}this.log(`        \u21B3 pairs budget ${d.minPairs}`+(d.configured?" (metadata.autoEvolvePairs)":" (default)")+` \xB7 writes ${l.autoEvolveApply===!0||l.autoEvolveApply==="true"?"ENABLED":"off (dry run)"}`),d.note&&this.log(`        ! ${d.note}`);try{let t=await y(`/api/eval-targets/${s.id}/promotions?limit=1`),a=t.promotions[0];if(a){let n=a.gates?.cost,f=n?.modelCalls?`${n.modelCalls} model calls, ${n.usablePairs??"?"} usable pair(s)`+(n.mde!=null?`, resolved +/-${ee(n.mde)}`:"")+(n.wallMs?`, ${(n.wallMs/6e4).toFixed(0)} min`:""):"cost not recorded (decision predates cost tracking)";this.log(`        \u21B3 last run ${new Date(a.createdAt).toISOString().slice(0,10)}: ${a.decision}${a.applied?" (APPLIED)":""} \u2014 ${f}`)}let o=t.counts??{};this.log(`        \u21B3 lifetime: ${o.promote??0} promote / ${o.revert??0} revert / ${o.refuse??0} refuse`)}catch{}}this.log("        \u21B3 pause the spend: metadata.autoEvolvePaused=true (keeps the opt-in)")}}};function A(h){if(h===null)return"never";if(h<90)return`${Math.round(h)}s`;let r=Math.round(h/60);return r<90?`${r}m`:`${Math.round(r/60)}h`}S(A,"formatAge");export{I as default};
+import{c as q,d as K}from"../chunk-TYFEWLYZ.js";import{b as L,c as W}from"../chunk-V5U3E4BZ.js";import{a as J}from"../chunk-2TJ7Q2OU.js";import{e as O,f as _,g as M}from"../chunk-QWCEZEDO.js";import"../chunk-KKDHD4XA.js";import"../chunk-SZ62CP55.js";import{d as oe,e as ne}from"../chunk-ICLAHFKK.js";import"../chunk-M7QSC64D.js";import"../chunk-DZRBB34J.js";import"../chunk-KR32Y5Q5.js";import"../chunk-HBJMELKB.js";import{i as z,j as Q,k as Z,m as ee,n as te}from"../chunk-GYKYJJNF.js";import"../chunk-G7V4ZZZ3.js";import"../chunk-AHLVY52I.js";import"../chunk-ULFSGFT2.js";import"../chunk-55HFAX5Y.js";import"../chunk-GWQ24FQP.js";import"../chunk-PPAHIVG5.js";import"../chunk-CT7FWSIU.js";import"../chunk-5MXZJWQS.js";import"../chunk-N7ODX6PO.js";import{a as V}from"../chunk-OZQNSELV.js";import"../chunk-HFGO5762.js";import"../chunk-VS5ROGDP.js";import"../chunk-UGA2NFRG.js";import"../chunk-VZYNZS5U.js";import"../chunk-U2NT5OSQ.js";import"../chunk-KGURGIZ2.js";import"../chunk-GYNZ3OZF.js";import"../chunk-UPFH72PF.js";import"../chunk-U6AASXY3.js";import"../chunk-CDY7TQFZ.js";import"../chunk-KFNFLMUR.js";import"../chunk-LARY6ZFT.js";import"../chunk-JP6ULOEX.js";import"../chunk-SVZYRGO5.js";import"../chunk-WULEBQPI.js";import{d as F}from"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import{a as X,b as D,c as Y,g as j,h as G}from"../chunk-QI3FJKGK.js";import{a as B}from"../chunk-TQFHRD5B.js";import"../chunk-IZTIYR5Q.js";import"../chunk-5LTJNYCP.js";import"../chunk-3EZZGA72.js";import"../chunk-XAGNGQL3.js";import"../chunk-3AV3GTMP.js";import{a as U,e as y}from"../chunk-5VWVQCNR.js";import"../chunk-MHLHLJ5T.js";import"../chunk-XQYAVSUQ.js";import{a as N,g as v,l as x}from"../chunk-PZNWH4VQ.js";import{a as S}from"../chunk-ZLLER3HP.js";import{spawnSync as le}from"node:child_process";import{existsSync as ce,readFileSync as ue}from"node:fs";import{join as se}from"node:path";var I=class h extends B{static{S(this,"Diagnostics")}static description='Diagnose capture / proxy / connectivity problems (e.g. a client-side "502 Bad Gateway")';static examples=["<%= config.bin %> diagnostics"];baseUrl(){return process.env.RULEMETRIC_API_URL??N().apiUrl??U}async checkDirect(){let r=process.env.HTTPS_PROXY,i=process.env.HTTP_PROXY;delete process.env.HTTPS_PROXY,delete process.env.HTTP_PROXY;let s=new AbortController,l=setTimeout(()=>s.abort(),8e3);try{return(await globalThis.fetch(`${this.baseUrl()}/health`,{signal:s.signal})).ok}catch{return!1}finally{clearTimeout(l),r!==void 0&&(process.env.HTTPS_PROXY=r),i!==void 0&&(process.env.HTTP_PROXY=i)}}configuredProxy(){if(process.env.HTTPS_PROXY)return process.env.HTTPS_PROXY;for(let r of["settings.local.json","settings.json"]){let i=se(process.cwd(),".claude",r);if(ce(i))try{let s=JSON.parse(ue(i,"utf8"));if(s.env?.HTTPS_PROXY)return s.env.HTTPS_PROXY}catch{}}return null}checkThroughProxy(r){let i=le("curl",["-sS","-x",r,"--max-time","8","-o","/dev/null","-w","%{http_code}",`${this.baseUrl()}/health`],{encoding:"utf8",stdio:["ignore","pipe","ignore"]});return i.error||typeof i.stdout!="string"?null:i.stdout.trim()==="200"}async run(){await this.parse(h);let r=process.version,i=this.configuredProxy(),s=i!=null,[l,m,d]=await Promise.all([this.checkDirect(),M(O),M(_)]),p=s&&i?this.checkThroughProxy(i):null,t=S((e,c,T)=>{let u=c===null?"\u2013":c?"[OK]":"[!!]";this.log(`  ${u} ${e.padEnd(22)} ${T}`)},"line");this.log("RuleMetric diagnostics"),this.log(""),t("Node",!L(r),`${r}${L(r)?" (< 20.11 \u2014 breaks the bundled proxy)":""}`),t("API (direct)",l,`${this.baseUrl()}/health ${l?"reachable":"UNREACHABLE"}`),t("Authenticated",v(),v()?"yes":"no \u2014 run `rulemetric auth login`"),t("Proxy configured",s?!0:null,s?i:"no HTTPS_PROXY set"),s&&(t("Gateway :"+O,m,m?"listening":"NOT listening"),t("mitmproxy :"+_,d,d?"listening":"not listening (gateway should route direct)"),t("API (via proxy)",p,p===null?"inconclusive (curl unavailable)":p?"reachable":"FAILED \u2014 502/blocked through your proxy"));let a=F();t("claude CLI",a.claudeBin!=null,a.claudeBin??"not found \u2014 LLM features (grading, distill) will no-op"),t("codex CLI",a.codexBin!=null?!0:null,a.codexBin?`${a.codexBin}${a.codexConfiguredModel?` (model ${a.codexConfiguredModel})`:""}`:"not found \u2014 optional; enables the cross-vendor reviewer (evals validate-judge)");let o=null;if(v()&&l)try{o=await y("/api/workers/health")}catch{o=null}if(o){let e=o.worker;t("Worker",!!e.active,e.present?e.active?`online (last heartbeat ${e.ageSeconds??"?"}s ago${e.clientVersion?`, v${e.clientVersion}`:""})`:`NOT RUNNING \u2014 last heartbeat ${A(e.ageSeconds)} ago`:"never connected \u2014 no worker has ever reported in");let c=o.worker.engines;if(c!=null){let u=[c.claude&&"claude",c.codex&&"codex"].filter(Boolean);t("LLM engine",u.length>0,u.length>0?`${u.join(" + ")} available to the worker`:"NONE \u2014 the worker found no `claude` and no `codex`. LLM jobs will complete with empty output rather than failing.")}let T=o.pending.reduce((u,E)=>u+E.count,0);if(T>0){let u=o.verdict.stuck.reduce(($,k)=>$+k.count,0),E=o.verdict.unclaimable.reduce(($,k)=>$+k.count,0),R=(o.verdict.engineBlocked??[]).reduce(($,k)=>$+k.count,0);t("Job queue",u===0&&R===0,`${T} pending`+(u>0?` \u2014 ${u} waiting on this worker`:"")+(R>0?` \u2014 ${R} need a local LLM engine`:"")+(E>0?` \u2014 ${E} need a worker with DATABASE_URL`:""))}else t("Job queue",!0,"empty")}else v()&&l&&t("Worker",null,"could not read /workers/health (older API?)");let n=o?.worker.machine;if(n){let e=n.suspends+n.stalls,c=n.lastPause;t("Worker pauses",e===0?!0:null,e===0?`none since ${n.since} \u2014 no sleep or stall seen by the worker`:`${n.suspends} suspend${n.suspends===1?"":"s"}`+(n.stalls>0?` + ${n.stalls} event-loop stall${n.stalls===1?"":"s"}`:"")+` since ${n.since}, ${A(Math.round(n.pausedMs/1e3))} paused total`+(c?`; last: ${c.kind} of ${A(Math.round(c.wallGapMs/1e3))} ending ${c.detectedAt}`:"")+" \u2014 jobs in flight during a suspend lose their lease and are redone on wake")}let f=await q();if(f){t("Machine sleep (pmset)",f.sleeps===0?!0:null,K(f));for(let e of f.recent.slice(-3)){let c=e.seconds!=null?` (${A(e.seconds)})`:"";this.log(`        \u21B3 ${e.at.toISOString()} ${e.kind}${c}: ${e.reason.slice(0,80)}`)}}let C=V(process.cwd()),g=await D({configDir:x()}),re=await Y({configDir:x()}),w=` \u2014 ${G(re)}`,b=J(process.cwd()),H=[b.user&&`user hooks (${b.user})`,b.workspace&&`workspace hooks (${b.workspace})`].filter(Boolean),ie={claude_code:`session hooks (+ proxy for instruction linking)${w}`,codex:`worker transcript sync of ~/.codex/sessions \u2014 ${j(g)}`,cursor:`user hooks (~/.cursor/hooks.json)${w}`,copilot_agent:`workspace hooks${w}`,antigravity:H.length>0?`${H.join(" + ")}${w}`:"NOT CAPTURED \u2014 no rulemetric block in ~/.gemini/config/hooks.json or .agents/hooks.json; run `rulemetric hooks install` (user hooks are on by default)"};this.log(""),this.log("  Capture coverage (agents found on this machine):");for(let e of C)this.log(`    \u2022 ${e.padEnd(15)} ${ie[e]??"captured"}`);if(C.includes("codex"))if(g.error)this.log(`    ! ${g.error}`);else if(g.mode==="allowlist")for(let e of g.roots)this.log(`        \u21B3 ${e}`);else g.mode==="all"&&this.log(`        \u21B3 scope it with ${se(x(),X)} ({"codex":{"roots":["/path/to/repo"]}}) or RULEMETRIC_CODEX_CAPTURE=off`);o&&!o.worker.active&&C.includes("codex")&&g.mode!=="off"&&this.log("    ! Codex capture runs INSIDE the worker \u2014 it is stopped while the worker is down."),v()&&l&&await this.reportEvolutionSpend();let P=W({nodeVersion:r,directApiOk:l,httpsProxySet:s,gatewayAlive:m,mitmproxyAlive:d,proxyPathOk:p});this.log("");let ae=P.severity==="ok"?"[OK]":P.severity==="warn"?"[!!]":"[XX]";this.log(`${ae} ${P.headline}`);for(let e of P.hints)this.log(`   \u2022 ${e}`);if(o&&o.verdict.severity!=="ok"){let e=o.verdict.severity==="warn"?"[!!]":"[XX]";this.log(""),this.log(`${e} ${o.verdict.headline}`);for(let c of o.verdict.hints)this.log(`   \u2022 ${c}`)}}async reportEvolutionSpend(){let r;try{r=await y("/api/eval-targets?type=instruction&limit=200")}catch{return}let i=r.filter(s=>{let l=s.metadata??{};return l.autoEvolveEnabled===!0||l.autoEvolveEnabled==="true"});if(i.length!==0){this.log(""),this.log("  Nightly instruction evolution (cron_instruction_evolution, 07:30 UTC):");for(let s of i){let l=s.metadata??{},m=Q(l),d=z(l);if(m){this.log(`    \u2022 ${s.name} \u2014 PAUSED (metadata.autoEvolvePaused). Opt-in preserved.`),this.log("        \u21B3 resume: set autoEvolvePaused=false");continue}let p=null;try{p=(await y(`/api/eval-targets/${s.id}/evals`)).filter(a=>!a.retiredAt).length}catch{}if(this.log(`    \u2022 ${s.name}`),p!=null){let t=ne(p,d.minPairs,oe(d,p));if(!t.feasible)this.log(`        \u21B3 cannot reach a verdict \u2014 ${t.reason}`);else{let a=Z({liveCases:p,passes:t.passes});this.log(`        \u21B3 each night: ${te(a)}`)}}this.log(`        \u21B3 pairs budget ${d.minPairs}`+(d.configured?" (metadata.autoEvolvePairs)":" (default)")+` \xB7 writes ${l.autoEvolveApply===!0||l.autoEvolveApply==="true"?"ENABLED":"off (dry run)"}`),d.note&&this.log(`        ! ${d.note}`);try{let t=await y(`/api/eval-targets/${s.id}/promotions?limit=1`),a=t.promotions[0];if(a){let n=a.gates?.cost,f=n?.modelCalls?`${n.modelCalls} model calls, ${n.usablePairs??"?"} usable pair(s)`+(n.mde!=null?`, resolved +/-${ee(n.mde)}`:"")+(n.wallMs?`, ${(n.wallMs/6e4).toFixed(0)} min`:""):"cost not recorded (decision predates cost tracking)";this.log(`        \u21B3 last run ${new Date(a.createdAt).toISOString().slice(0,10)}: ${a.decision}${a.applied?" (APPLIED)":""} \u2014 ${f}`)}let o=t.counts??{};this.log(`        \u21B3 lifetime: ${o.promote??0} promote / ${o.revert??0} revert / ${o.refuse??0} refuse`)}catch{}}this.log("        \u21B3 pause the spend: metadata.autoEvolvePaused=true (keeps the opt-in)")}}};function A(h){if(h===null)return"never";if(h<90)return`${Math.round(h)}s`;let r=Math.round(h/60);return r<90?`${r}m`:`${Math.round(r/60)}h`}S(A,"formatAge");export{I as default};

package/dist/commands/eval-targets/create.js

@@ -1 +1 @@
-import{a}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{f as s}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as o}from"../../chunk-ZLLER3HP.js";import{Flags as e}from"@oclif/core";import{readFileSync as p}from"node:fs";var n=class c extends a{static{o(this,"EvalTargetCreate")}static description="Create an eval target (instruction, skill, prompt, or plan to test)";static examples=["<%= config.bin %> eval-targets create --name my-rules --type instruction --file ./CLAUDE.md",'<%= config.bin %> eval-targets create --name my-prompt --type prompt --content "Always use TypeScript"'];static flags={name:e.string({char:"n",description:"Name for the eval target",required:!0}),type:e.string({char:"t",description:"Target type",options:["instruction","skill","prompt","plan"],required:!0}),file:e.string({char:"f",description:"Path to content file"}),content:e.string({char:"c",description:"Content string (alternative to --file)"}),description:e.string({char:"d",description:"Description of the target",default:""})};async run(){let{flags:t}=await this.parse(c);this.requireAuth();let i;t.file?i=p(t.file,"utf-8"):t.content?i=t.content:this.error("Provide either --file or --content");let r=await s("/api/eval-targets",{name:t.name,type:t.type,content:i,description:t.description});this.log(`Created eval target: ${r.name} (${r.id}) v${r.version}`)}};export{n as default};
+import{a}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{f as s}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as o}from"../../chunk-ZLLER3HP.js";import{Flags as e}from"@oclif/core";import{readFileSync as p}from"node:fs";var n=class c extends a{static{o(this,"EvalTargetCreate")}static description="Create an eval target (instruction, skill, prompt, or plan to test)";static examples=["<%= config.bin %> eval-targets create --name my-rules --type instruction --file ./CLAUDE.md",'<%= config.bin %> eval-targets create --name my-prompt --type prompt --content "Always use TypeScript"'];static flags={name:e.string({char:"n",description:"Name for the eval target",required:!0}),type:e.string({char:"t",description:"Target type",options:["instruction","skill","prompt","plan"],required:!0}),file:e.string({char:"f",description:"Path to content file"}),content:e.string({char:"c",description:"Content string (alternative to --file)"}),description:e.string({char:"d",description:"Description of the target",default:""})};async run(){let{flags:t}=await this.parse(c);this.requireAuth();let i;t.file?i=p(t.file,"utf-8"):t.content?i=t.content:this.error("Provide either --file or --content");let r=await s("/api/eval-targets",{name:t.name,type:t.type,content:i,description:t.description});this.log(`Created eval target: ${r.name} (${r.id}) v${r.version}`)}};export{n as default};

package/dist/commands/eval-targets/list.js

@@ -1 +1 @@
-import{a as p,b as e}from"../../chunk-W2MV25SB.js";import{a as o}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as s}from"../../chunk-ZLLER3HP.js";import{Flags as m}from"@oclif/core";import{printTable as d}from"@oclif/table";var r=class l extends o{static{s(this,"EvalTargetList")}static description="List eval targets";static examples=["<%= config.bin %> eval-targets list","<%= config.bin %> eval-targets list --type instruction"];static flags={type:m.string({char:"t",description:"Filter by type",options:["instruction","skill","prompt","plan"]})};async run(){let{flags:i}=await this.parse(l);this.requireAuth();let c=i.type?`?type=${i.type}`:"",a=await n(`/api/eval-targets${c}`);if(a.length===0){this.log("No eval targets found. Create one with `rulemetric eval-targets create`.");return}d({data:a.map(t=>({Name:e(t.name,30),Type:t.type,Version:`v${t.version}`,Description:e(t.description,40),Updated:p(t.updatedAt)})),columns:["Name","Type","Version","Description","Updated"]})}};export{r as default};
+import{a as p,b as e}from"../../chunk-W2MV25SB.js";import{a as o}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as s}from"../../chunk-ZLLER3HP.js";import{Flags as m}from"@oclif/core";import{printTable as d}from"@oclif/table";var r=class l extends o{static{s(this,"EvalTargetList")}static description="List eval targets";static examples=["<%= config.bin %> eval-targets list","<%= config.bin %> eval-targets list --type instruction"];static flags={type:m.string({char:"t",description:"Filter by type",options:["instruction","skill","prompt","plan"]})};async run(){let{flags:i}=await this.parse(l);this.requireAuth();let c=i.type?`?type=${i.type}`:"",a=await n(`/api/eval-targets${c}`);if(a.length===0){this.log("No eval targets found. Create one with `rulemetric eval-targets create`.");return}d({data:a.map(t=>({Name:e(t.name,30),Type:t.type,Version:`v${t.version}`,Description:e(t.description,40),Updated:p(t.updatedAt)})),columns:["Name","Type","Version","Description","Updated"]})}};export{r as default};

package/dist/commands/evals/agent.js

@@ -1,3 +1,3 @@
-import{a as G,b as V,c as j,d as q}from"../../chunk-J4W6GNCF.js";import{b as M}from"../../chunk-GH4EZK2O.js";import{c as g}from"../../chunk-HTAHNQZJ.js";import{e as D}from"../../chunk-3MYCWZ63.js";import"../../chunk-5TM7AOH2.js";import"../../chunk-IEQDLURA.js";import"../../chunk-KINIFUZQ.js";import"../../chunk-KOORMTZO.js";import"../../chunk-Q7IDLOSP.js";import"../../chunk-2PO5I3XR.js";import"../../chunk-TQM7MF5C.js";import"../../chunk-RDQXK5LG.js";import"../../chunk-QXPGTEMO.js";import"../../chunk-ZYGBABMZ.js";import"../../chunk-GIQ2HK3W.js";import"../../chunk-STUNT4N3.js";import"../../chunk-E4CIJ67Z.js";import"../../chunk-FZMO7XSN.js";import"../../chunk-MOJY6S5J.js";import"../../chunk-KUW6AVSI.js";import"../../chunk-5A6TMSRO.js";import"../../chunk-FCALFMZV.js";import"../../chunk-JU5ZNXPK.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-VBGHPAUT.js";import"../../chunk-GGUXEZC3.js";import"../../chunk-ZLBYYLJZ.js";import"../../chunk-USKRCVX3.js";import"../../chunk-7YZURVZD.js";import"../../chunk-SW24KZ5C.js";import"../../chunk-QPAOX7P2.js";import"../../chunk-3ZHVP3KE.js";import"../../chunk-EFEPD4AZ.js";import"../../chunk-HOQPVOMF.js";import"../../chunk-AKCCNLHL.js";import"../../chunk-CVPAKEEZ.js";import"../../chunk-VUNJGZPP.js";import"../../chunk-GLBWNB2N.js";import"../../chunk-CQD2R3BA.js";import"../../chunk-3UKZJAFS.js";import"../../chunk-77FH6UZP.js";import"../../chunk-KE5K3F7J.js";import"../../chunk-WL34G6IR.js";import"../../chunk-KR36ZDGN.js";import"../../chunk-GIRPEMTW.js";import"../../chunk-DQYYHX6K.js";import"../../chunk-4ZDE63UG.js";import"../../chunk-F5GSKVF2.js";import"../../chunk-PIVEJTNF.js";import"../../chunk-FHRBW4AQ.js";import"../../chunk-ZBTJ5R7C.js";import"../../chunk-DANMUHNW.js";import"../../chunk-ISYU5BMU.js";import"../../chunk-CPDW7WW2.js";import"../../chunk-CDSSOOIG.js";import"../../chunk-XFOG2PDJ.js";import"../../chunk-SIGZEQ3K.js";import"../../chunk-FXQS7AEZ.js";import"../../chunk-LJRE3LZ2.js";import"../../chunk-TSLBCZ5P.js";import"../../chunk-65LFPUDB.js";import"../../chunk-HBDKOU2N.js";import"../../chunk-3H44B6PB.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-MODXEGCU.js";import"../../chunk-MHVHGKDK.js";import"../../chunk-RBSEE2Z6.js";import"../../chunk-PDXHRF4E.js";import"../../chunk-KGU2IW54.js";import"../../chunk-M7QSC64D.js";import"../../chunk-ZKPURXUS.js";import"../../chunk-ISEUEYSC.js";import"../../chunk-GYKYJJNF.js";import"../../chunk-G7V4ZZZ3.js";import"../../chunk-LJE5J3EW.js";import"../../chunk-AHLVY52I.js";import"../../chunk-YXMAGC53.js";import"../../chunk-T7HZZ4DJ.js";import"../../chunk-DHIEGG4T.js";import"../../chunk-PPAHIVG5.js";import"../../chunk-5MXZJWQS.js";import"../../chunk-CT7FWSIU.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-OZQNSELV.js";import"../../chunk-HFGO5762.js";import"../../chunk-VS5ROGDP.js";import"../../chunk-FADINV4H.js";import"../../chunk-SWSOEYC7.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-FG343JM3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-KFNFLMUR.js";import"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{b as A,d as C,g as S,i as E,j as L,k as $}from"../../chunk-QI3FJKGK.js";import{a as O}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{f as m}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import{e as I,l as p}from"../../chunk-PZNWH4VQ.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as de,context as me}from"@opentelemetry/api";import{NodeSDK as W}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as Y}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as J,ConsoleSpanExporter as z,SimpleSpanProcessor as Q}from"@opentelemetry/sdk-trace-node";import{Resource as Z}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as ee,ATTR_SERVICE_VERSION as te}from"@opentelemetry/semantic-conventions";function F(r){if(!r)return{};let n={};for(let o of r.split(",")){let s=o.indexOf("=");if(s===-1)continue;let i=o.slice(0,s).trim(),c=o.slice(s+1).trim();i&&(n[i]=c)}return n}e(F,"parseOtlpHeaders");var u=null;function T(){return process.env.TELEMETRY_ENABLED!=="false"}e(T,"isTelemetryEnabled");function x(r){if(!T()||u!==null)return"off";let n=new Z({[ee]:r.serviceName,[te]:r.serviceVersion??"0.1.0"}),o=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,s,i;if(o){let c=F(process.env.OTEL_EXPORTER_OTLP_HEADERS),l=new Y({url:`${o}/v1/traces`,headers:c});i=new J(l),s="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")i=void 0,s="silent";else{let c=new z;i=new Q(c),s="console"}return u=new W({resource:n,spanProcessors:i?[i]:[]}),u.start(),s}e(x,"initTelemetry");async function h(){u&&(await u.shutdown(),u=null)}e(h,"shutdownTelemetry");import{SpanKind as Pe,SpanStatusCode as Re}from"@opentelemetry/api";import{trace as Ie,context as Oe}from"@opentelemetry/api";import*as f from"@sentry/node";var v=!1;function w(r){if(v)return!1;let n=process.env.SENTRY_DSN;return n?(f.init({dsn:n,serverName:r.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),v=!0,!0):!1}e(w,"initSentry");async function y(r){v&&await f.flush(r)}e(y,"flushSentry");import{existsSync as ne,mkdirSync as oe,readFileSync as k,unlinkSync as X,writeFileSync as U}from"node:fs";import{dirname as se,join as R}from"node:path";import{execFileSync as ie}from"node:child_process";var a=R(p(),"evals-agent.lock");function ce(r){try{return process.kill(r,0),!0}catch(n){return n.code==="EPERM"}}e(ce,"pidAlive");function ae(r){if(!ce(r))return!1;try{let n=ie("ps",["-p",String(r),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(n)}catch{return!0}}e(ae,"isLiveAgent");function le(){try{oe(se(a),{recursive:!0})}catch{}for(let r=0;r<2;r++)try{return U(a,String(process.pid),{flag:"wx"}),{ok:!0}}catch(n){if(n.code!=="EEXIST")throw n;let o=0;try{o=Number(k(a,"utf-8").trim())}catch{}if(o&&o!==process.pid&&ae(o))return{ok:!1,holder:o};try{X(a)}catch{}}try{U(a,String(process.pid))}catch{}return{ok:!0}}e(le,"acquireAgentLock");function P(){try{ne(a)&&Number(k(a,"utf-8").trim())===process.pid&&X(a)}catch{}}e(P,"releaseAgentLock");var N=class r extends O{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(r),this.requireAuth();let n=le();if(!n.ok){this.log(`Another rulemetric evals agent is already running (pid ${n.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",w({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let s=x({serviceName:"rulemetric-agent"});s==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):s==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!I()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
+import{a as G,b as V,c as j,d as q}from"../../chunk-4LVSD2XZ.js";import{b as M}from"../../chunk-GH4EZK2O.js";import{c as g}from"../../chunk-BDWQ6J4V.js";import{e as D}from"../../chunk-43TRZXJP.js";import"../../chunk-5U5SHJJA.js";import"../../chunk-IEQDLURA.js";import"../../chunk-KINIFUZQ.js";import"../../chunk-QRKMRG6W.js";import"../../chunk-D7L7IMJQ.js";import"../../chunk-BQVUZ6VE.js";import"../../chunk-QJZ4DMVQ.js";import"../../chunk-URUHD4OI.js";import"../../chunk-JU5ZNXPK.js";import"../../chunk-YGFZ6BLU.js";import"../../chunk-BCPIY47G.js";import"../../chunk-NLUHMHPM.js";import"../../chunk-FTCC4OZ4.js";import"../../chunk-WLLIFRRE.js";import"../../chunk-RRBFKVCI.js";import"../../chunk-ZYGBABMZ.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-NSO2EBYE.js";import"../../chunk-3ZHVP3KE.js";import"../../chunk-2C7CFCMM.js";import"../../chunk-HOQPVOMF.js";import"../../chunk-WSUKOU3X.js";import"../../chunk-A6XQSO62.js";import"../../chunk-MOJY6S5J.js";import"../../chunk-JGJU4JX5.js";import"../../chunk-VBGHPAUT.js";import"../../chunk-IPNBW7ZW.js";import"../../chunk-GQYF2WVE.js";import"../../chunk-274MXEKI.js";import"../../chunk-ANLPPIMS.js";import"../../chunk-AZRGURHR.js";import"../../chunk-N2PUNIWM.js";import"../../chunk-MJLW5KN2.js";import"../../chunk-SW24KZ5C.js";import"../../chunk-VCWT5KZL.js";import"../../chunk-LELSVLNK.js";import"../../chunk-A2JFQUDW.js";import"../../chunk-LZPE6424.js";import"../../chunk-3UKZJAFS.js";import"../../chunk-77FH6UZP.js";import"../../chunk-H4VIQHBV.js";import"../../chunk-4GM4OEQY.js";import"../../chunk-GIRPEMTW.js";import"../../chunk-DQYYHX6K.js";import"../../chunk-RWQG25JU.js";import"../../chunk-3ZHBOILE.js";import"../../chunk-PIVEJTNF.js";import"../../chunk-BOYWPUV2.js";import"../../chunk-ENW5SUQJ.js";import"../../chunk-C3XWDYZJ.js";import"../../chunk-CAQLSYOK.js";import"../../chunk-RQBN7WNS.js";import"../../chunk-ZJABRHIP.js";import"../../chunk-XFOG2PDJ.js";import"../../chunk-SIGZEQ3K.js";import"../../chunk-AZWY2DOT.js";import"../../chunk-LJRE3LZ2.js";import"../../chunk-FD53AP7T.js";import"../../chunk-DOEPIICK.js";import"../../chunk-VNGJRYBD.js";import"../../chunk-FLXWL4F4.js";import"../../chunk-YT25HLWB.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-MODXEGCU.js";import"../../chunk-MHVHGKDK.js";import"../../chunk-RBSEE2Z6.js";import"../../chunk-TL57YUJV.js";import"../../chunk-ICLAHFKK.js";import"../../chunk-M7QSC64D.js";import"../../chunk-DZRBB34J.js";import"../../chunk-KR32Y5Q5.js";import"../../chunk-HBJMELKB.js";import"../../chunk-GYKYJJNF.js";import"../../chunk-G7V4ZZZ3.js";import"../../chunk-AHLVY52I.js";import"../../chunk-ULFSGFT2.js";import"../../chunk-55HFAX5Y.js";import"../../chunk-GWQ24FQP.js";import"../../chunk-PPAHIVG5.js";import"../../chunk-CT7FWSIU.js";import"../../chunk-5MXZJWQS.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-OZQNSELV.js";import"../../chunk-HFGO5762.js";import"../../chunk-VS5ROGDP.js";import"../../chunk-UGA2NFRG.js";import"../../chunk-VZYNZS5U.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-U6AASXY3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-KFNFLMUR.js";import"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{b as A,d as C,g as S,i as E,j as L,k as $}from"../../chunk-QI3FJKGK.js";import{a as O}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{f as m}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import{e as I,l as p}from"../../chunk-PZNWH4VQ.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as de,context as me}from"@opentelemetry/api";import{NodeSDK as W}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as Y}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as J,ConsoleSpanExporter as z,SimpleSpanProcessor as Q}from"@opentelemetry/sdk-trace-node";import{Resource as Z}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as ee,ATTR_SERVICE_VERSION as te}from"@opentelemetry/semantic-conventions";function F(r){if(!r)return{};let n={};for(let o of r.split(",")){let s=o.indexOf("=");if(s===-1)continue;let i=o.slice(0,s).trim(),c=o.slice(s+1).trim();i&&(n[i]=c)}return n}e(F,"parseOtlpHeaders");var u=null;function T(){return process.env.TELEMETRY_ENABLED!=="false"}e(T,"isTelemetryEnabled");function x(r){if(!T()||u!==null)return"off";let n=new Z({[ee]:r.serviceName,[te]:r.serviceVersion??"0.1.0"}),o=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,s,i;if(o){let c=F(process.env.OTEL_EXPORTER_OTLP_HEADERS),l=new Y({url:`${o}/v1/traces`,headers:c});i=new J(l),s="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")i=void 0,s="silent";else{let c=new z;i=new Q(c),s="console"}return u=new W({resource:n,spanProcessors:i?[i]:[]}),u.start(),s}e(x,"initTelemetry");async function h(){u&&(await u.shutdown(),u=null)}e(h,"shutdownTelemetry");import{SpanKind as Pe,SpanStatusCode as Re}from"@opentelemetry/api";import{trace as Ie,context as Oe}from"@opentelemetry/api";import*as f from"@sentry/node";var v=!1;function w(r){if(v)return!1;let n=process.env.SENTRY_DSN;return n?(f.init({dsn:n,serverName:r.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),v=!0,!0):!1}e(w,"initSentry");async function y(r){v&&await f.flush(r)}e(y,"flushSentry");import{existsSync as ne,mkdirSync as oe,readFileSync as k,unlinkSync as X,writeFileSync as U}from"node:fs";import{dirname as se,join as R}from"node:path";import{execFileSync as ie}from"node:child_process";var a=R(p(),"evals-agent.lock");function ce(r){try{return process.kill(r,0),!0}catch(n){return n.code==="EPERM"}}e(ce,"pidAlive");function ae(r){if(!ce(r))return!1;try{let n=ie("ps",["-p",String(r),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(n)}catch{return!0}}e(ae,"isLiveAgent");function le(){try{oe(se(a),{recursive:!0})}catch{}for(let r=0;r<2;r++)try{return U(a,String(process.pid),{flag:"wx"}),{ok:!0}}catch(n){if(n.code!=="EEXIST")throw n;let o=0;try{o=Number(k(a,"utf-8").trim())}catch{}if(o&&o!==process.pid&&ae(o))return{ok:!1,holder:o};try{X(a)}catch{}}try{U(a,String(process.pid))}catch{}return{ok:!0}}e(le,"acquireAgentLock");function P(){try{ne(a)&&Number(k(a,"utf-8").trim())===process.pid&&X(a)}catch{}}e(P,"releaseAgentLock");var N=class r extends O{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(r),this.requireAuth();let n=le();if(!n.ok){this.log(`Another rulemetric evals agent is already running (pid ${n.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",w({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let s=x({serviceName:"rulemetric-agent"});s==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):s==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!I()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
 `);let c=new AbortController,l=await A({configDir:p()});if(this.log(`[transcript-sync] Codex capture: ${S(l)}`),l.error)this.log(`[transcript-sync] WARNING: ${l.error}`);else if(l.mode==="allowlist")for(let t of l.roots)this.log(`[transcript-sync]   \u2022 ${t}`);let d=await C({configDir:p()});if(this.log(`[transcript-sync] Pi capture: ${E(d)}`),d.error)this.log(`[transcript-sync] WARNING: ${d.error}`);else if(d.mode==="allowlist")for(let t of d.roots)this.log(`[transcript-sync]   \u2022 ${t}`);let B=new M({postSnapshot:e((t,K)=>m(`/api/sessions/${t}/context-snapshots`,K),"postSnapshot"),linkInstructions:e(t=>m("/api/sessions/link-instructions",{sessionId:t}),"linkInstructions"),log:e(t=>this.log(`[transcript-sync] ${t}`),"log")}),H=new G({providers:[V(void 0,void 0,L({configDir:p(),onChange:e(t=>this.log(`[transcript-sync] Codex capture changed: ${S(t)}`),"onChange")})),j(void 0,void 0,$({configDir:p(),onChange:e(t=>this.log(`[transcript-sync] Pi capture changed: ${E(t)}`),"onChange")}))],statePath:R(p(),"transcript-sync-state.json"),importTranscript:e(t=>m("/api/sessions/import",t),"importTranscript"),emitInstructions:e(t=>B.emit(t),"emitInstructions"),log:e(t=>this.log(`[transcript-sync] ${t}`),"log")});q(H,{signal:c.signal,log:e(t=>this.log(`[transcript-sync] ${t}`),"log")});let _=e(async t=>{this.log(`
 [${t}] shutting down agent...`),c.abort(),await Promise.allSettled([y(2e3),h(),g()]),P(),process.exit(0)},"shutdown");process.on("SIGINT",()=>{_("SIGINT")}),process.on("SIGTERM",()=>{_("SIGTERM")});let b=!1;try{await D({signal:c.signal,log:e(t=>this.log(t),"log"),version:this.config.version,readInstalledVersion:e(()=>{try{return JSON.parse(k(R(this.config.root,"package.json"),"utf-8")).version??null}catch{return null}},"readInstalledVersion"),onVersionDrift:e(()=>{b=!0},"onVersionDrift")}),b&&(this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code."),await Promise.allSettled([y(2e3),h(),g()]),P(),process.exit(1))}finally{P(),await g()}}};export{N as default};

package/dist/commands/evals/auto-run.js

@@ -1,4 +1,4 @@
-import{a as k}from"../../chunk-RBSEE2Z6.js";import{e as $}from"../../chunk-FADINV4H.js";import"../../chunk-SWSOEYC7.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-FG343JM3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as w}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as A,Flags as r}from"@oclif/core";var m=class y extends w{static{h(this,"EvalAutoRun")}static description="Run evals in a loop until the effect verdict is statistically powered (\xA76f auto-run-to-power)";static examples=["<%= config.bin %> evals auto-run <target-id>            # preview (cost + plan)","<%= config.bin %> evals auto-run <target-id> --yes      # run until powered / budget","<%= config.bin %> evals auto-run <target-id> --yes --max-runs 120 --batch-parallel 4"];static args={targetId:A.string({description:"Eval target ID",required:!0})};static flags={yes:r.boolean({char:"y",description:"Actually run. Without this, only previews the plan + cost."}),"max-runs":r.integer({description:"Hard ceiling on completed runs",default:200}),"batch-parallel":r.integer({description:"Concurrent executions within a batch",default:3}),timeout:r.integer({description:"Timeout per eval in ms",default:12e4}),model:r.string({description:"Model for eval execution"}),"grader-model":r.string({description:"Model for grading"}),"skip-baseline":r.boolean({description:"Skip without_target runs (no delta \u2014 not recommended)",default:!1})};async run(){let{args:C,flags:e}=await this.parse(y);this.requireAuth();let i=C.targetId,c=20,a=await n(`/api/eval-targets/${i}`),l=await n(`/api/eval-targets/${i}/evals`);l.length===0&&this.error(`No active cases on "${a.name}". Generate some (\`evals create\` / the web Generate button) or run \`evals refine-cases\` first.`);let b=l.length*(e["skip-baseline"]?1:2),f=Math.ceil(e["max-runs"]/b);if(this.log(`Auto-run "${a.name}" v${a.version} toward a powered \xB1${c}pp verdict.`),this.log(`Plan: up to ${e["max-runs"]} runs (${l.length} active case(s) \xD7 ${e["skip-baseline"]?1:2}/batch = ${b}/batch, \u2264${f} batches). Each run is a synthetic \`claude -p\` execution + grade.`),this.log('Note: this is a controlled synthetic A/B. For a DEPLOYED instruction with real usage, the Insights observational verdict is cheaper and uses real sessions \u2014 auto-run is for pre-deploy, catalog skills, or targets with no real "without" cohort.'),!e.yes){this.log(`
+import{a as k}from"../../chunk-RBSEE2Z6.js";import{e as $}from"../../chunk-UGA2NFRG.js";import"../../chunk-VZYNZS5U.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-U6AASXY3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as w}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as n}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as A,Flags as r}from"@oclif/core";var m=class y extends w{static{h(this,"EvalAutoRun")}static description="Run evals in a loop until the effect verdict is statistically powered (\xA76f auto-run-to-power)";static examples=["<%= config.bin %> evals auto-run <target-id>            # preview (cost + plan)","<%= config.bin %> evals auto-run <target-id> --yes      # run until powered / budget","<%= config.bin %> evals auto-run <target-id> --yes --max-runs 120 --batch-parallel 4"];static args={targetId:A.string({description:"Eval target ID",required:!0})};static flags={yes:r.boolean({char:"y",description:"Actually run. Without this, only previews the plan + cost."}),"max-runs":r.integer({description:"Hard ceiling on completed runs",default:200}),"batch-parallel":r.integer({description:"Concurrent executions within a batch",default:3}),timeout:r.integer({description:"Timeout per eval in ms",default:12e4}),model:r.string({description:"Model for eval execution"}),"grader-model":r.string({description:"Model for grading"}),"skip-baseline":r.boolean({description:"Skip without_target runs (no delta \u2014 not recommended)",default:!1})};async run(){let{args:C,flags:e}=await this.parse(y);this.requireAuth();let i=C.targetId,c=20,a=await n(`/api/eval-targets/${i}`),l=await n(`/api/eval-targets/${i}/evals`);l.length===0&&this.error(`No active cases on "${a.name}". Generate some (\`evals create\` / the web Generate button) or run \`evals refine-cases\` first.`);let b=l.length*(e["skip-baseline"]?1:2),f=Math.ceil(e["max-runs"]/b);if(this.log(`Auto-run "${a.name}" v${a.version} toward a powered \xB1${c}pp verdict.`),this.log(`Plan: up to ${e["max-runs"]} runs (${l.length} active case(s) \xD7 ${e["skip-baseline"]?1:2}/batch = ${b}/batch, \u2264${f} batches). Each run is a synthetic \`claude -p\` execution + grade.`),this.log('Note: this is a controlled synthetic A/B. For a DEPLOYED instruction with real usage, the Insights observational verdict is cheaper and uses real sessions \u2014 auto-run is for pre-deploy, catalog skills, or targets with no real "without" cohort.'),!e.yes){this.log(`
 Preview only. Re-run with --yes to start.`);return}let t={runsCompleted:0,maxRuns:e["max-runs"],batchesRun:0,maxBatches:f},x={targetId:i,targetVersion:a.version,iteration:1,timeout:e.timeout,model:e.model,graderModel:e["grader-model"],skipGrading:!1,targetType:a.type,targetName:a.name};for(;;){let s;try{s=await n(`/api/eval-targets/${i}/judge-calibration`)}catch(o){this.log(`
 \u25A0 stopping \u2014 could not read the power check: ${o instanceof Error?o.message:String(o)}`);break}let E={powered:s.powerCheck?.powered??!1,mde:s.powerCheck?.mde==null?1/0:s.powerCheck.mde},B=s.caseDiscrimination??{total:0,discriminating:0,nonDiscriminating:0,unpaired:0},p=k(E,B,t,{targetEffectPp:c});if(this.log(`
 ${p.action==="stop"?"\u25A0":"\u25B8"} ${p.reason}`),p.action==="stop")break;let v=await n(`/api/eval-targets/${i}/evals`);if(v.length===0){this.log("\u25A0 no active cases left \u2014 stopping.");break}let{completed:g,failed:D}=await $({cases:v,targetContent:a.content,concurrency:e["batch-parallel"],runNumber:t.batchesRun+1,opts:x,skipBaseline:e["skip-baseline"],onLog:h(o=>this.log(o),"onLog")});if(t.runsCompleted+=g,t.batchesRun+=1,this.log(`Batch ${t.batchesRun}: ${g} completed, ${D} failed \u2014 ${t.runsCompleted}/${t.maxRuns} runs, ${t.batchesRun}/${t.maxBatches} batches.`),g===0){this.log("\u25A0 batch produced 0 completed runs \u2014 the executor appears broken (check the model / `claude -p`). Aborting.");break}}let d=await n(`/api/eval-targets/${i}/judge-calibration`),u=d.powerCheck?.mde,R=u==null||!Number.isFinite(u)?"unknown":`\xB1${Math.round(u*100)}pp`;this.log(`

package/dist/commands/evals/benchmark.js

@@ -1,4 +1,4 @@
-import{b as h,c as f}from"../../chunk-7LE2W3WP.js";import"../../chunk-KFNFLMUR.js";import{a as m}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as c}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as u}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as v}from"@oclif/core";var d=class b extends m{static{u(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={iteration:v.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:o,flags:l}=await this.parse(b);this.requireAuth();let g=await c(`/api/eval-targets/${o.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${l.iteration}`);let a=await h(o.targetId,l.iteration);a.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
+import{b as h,c as f}from"../../chunk-7N55TWYQ.js";import"../../chunk-KFNFLMUR.js";import{a as m}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as c}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as u}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as v}from"@oclif/core";var d=class b extends m{static{u(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={iteration:v.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:o,flags:l}=await this.parse(b);this.requireAuth();let g=await c(`/api/eval-targets/${o.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${l.iteration}`);let a=await h(o.targetId,l.iteration);a.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
 Aggregated ${a.totalRuns} runs
 `);for(let[e,n]of Object.entries(a.runSummary)){this.log(`${e}:`);for(let[r,i]of Object.entries(n)){let s=i;this.log(`  ${r}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`)}}let t=a.verdict;if(t)if(this.log(`
 Verdict (${t.mode}, baseline=${t.baseline}):`),!t.sufficient)this.log(`  ${t.insufficientReason??"insufficient data"} \u2014 no effect claimed`);else{let e=t.delta>=0?"+":"";this.log(`  pass-rate delta: ${e}${(t.delta*100).toFixed(0)}% (n=${t.nWith} paired)`),this.log(`  95% CI: ${(t.ci.lower*100).toFixed(0)}\u2013${(t.ci.upper*100).toFixed(0)}%`),t.pValue!=null&&this.log(`  McNemar p: ${t.pValue.toFixed(4)}`),this.log(`  significant: ${t.significant?"YES":"no"}`)}if(a.notes.length>0){this.log(`

package/dist/commands/evals/compare.js

@@ -1,4 +1,4 @@
-import{a as $,b as A}from"../../chunk-647M2I2Y.js";import{f as p}from"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as f}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as l,f as w}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as B,Flags as b}from"@oclif/core";var c=class I extends f{static{h(this,"EvalCompareCommand")}static description="Blind A/B comparison between two eval target versions";static examples=["<%= config.bin %> evals compare <target-a> <target-b>"];static args={targetA:B.string({description:"First eval target ID",required:!0}),targetB:B.string({description:"Second eval target ID",required:!0})};static flags={model:b.string({description:"Model for execution and comparison"}),timeout:b.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:g,flags:o}=await this.parse(I);this.requireAuth();let[e,r]=await Promise.all([l(`/api/eval-targets/${g.targetA}`),l(`/api/eval-targets/${g.targetB}`)]),m=await l(`/api/eval-targets/${g.targetA}/evals`);m.length===0&&this.error("No evals found for target A. Create some first."),this.log(`Comparing "${e.name}" v${e.version} vs "${r.name}" v${r.version}`),this.log(`Running ${m.length} eval(s)...
+import{a as $,b as A}from"../../chunk-647M2I2Y.js";import{g as p}from"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as f}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as l,f as w}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as h}from"../../chunk-ZLLER3HP.js";import{Args as B,Flags as b}from"@oclif/core";var c=class I extends f{static{h(this,"EvalCompareCommand")}static description="Blind A/B comparison between two eval target versions";static examples=["<%= config.bin %> evals compare <target-a> <target-b>"];static args={targetA:B.string({description:"First eval target ID",required:!0}),targetB:B.string({description:"Second eval target ID",required:!0})};static flags={model:b.string({description:"Model for execution and comparison"}),timeout:b.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:g,flags:o}=await this.parse(I);this.requireAuth();let[e,r]=await Promise.all([l(`/api/eval-targets/${g.targetA}`),l(`/api/eval-targets/${g.targetB}`)]),m=await l(`/api/eval-targets/${g.targetA}/evals`);m.length===0&&this.error("No evals found for target A. Create some first."),this.log(`Comparing "${e.name}" v${e.version} vs "${r.name}" v${r.version}`),this.log(`Running ${m.length} eval(s)...
 `);let a=Math.random()>.5,E=a?r.content:e.content,T=a?e.content:r.content,C=a?r.name:e.name,x=a?e.name:r.name,v=[];for(let t of m){this.log(`[${t.name}] Running A...`);let s=await p({prompt:t.prompt,targetContent:E,timeout:o.timeout,model:o.model});this.log(`[${t.name}] Running B...`);let d=await p({prompt:t.prompt,targetContent:T,timeout:o.timeout,model:o.model});(s.error||d.error)&&this.error(`[${t.name}] execution failed (A: ${s.error??"ok"} | B: ${d.error??"ok"}) \u2014 aborting before the blind comparison`),v.push({evalName:t.name,outputA:s.output,outputB:d.output})}this.log(`
 Running blind comparison...`);let N=v.map(t=>`
 ## Eval: ${t.evalName}

package/dist/commands/evals/create.js

@@ -1 +1 @@
-import{a as c}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{f as i}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as m}from"../../chunk-ZLLER3HP.js";import{Args as f,Flags as r}from"@oclif/core";import{readFileSync as g}from"node:fs";var s=class d extends c{static{m(this,"EvalCreate")}static description="Create test cases (evals) for an eval target";static examples=['<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',"<%= config.bin %> evals create <target-id> --from-file evals.json"];static args={targetId:f.string({description:"Eval target ID",required:!0})};static flags={name:r.string({char:"n",description:"Eval name"}),prompt:r.string({char:"p",description:"Test prompt"}),expectations:r.string({char:"e",description:"Assertion (repeatable)",multiple:!0}),"from-file":r.string({description:"Bulk create from JSON file"})};async run(){let{args:n,flags:e}=await this.parse(d);if(this.requireAuth(),e["from-file"]){let a=JSON.parse(g(e["from-file"],"utf-8")),p=Array.isArray(a)?a:a.evals;for(let t of p){let l=await i(`/api/eval-targets/${n.targetId}/evals`,{name:t.name,prompt:t.prompt,expectations:t.expectations??[],files:t.files??[]});this.log(`Created eval: ${l.name} (${l.id})`)}this.log(`Created ${p.length} evals.`);return}(!e.name||!e.prompt)&&this.error("Provide --name and --prompt, or use --from-file for bulk creation");let o=await i(`/api/eval-targets/${n.targetId}/evals`,{name:e.name,prompt:e.prompt,expectations:e.expectations??[]});this.log(`Created eval: ${o.name} (${o.id})`)}};export{s as default};
+import{a as c}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{f as i}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as m}from"../../chunk-ZLLER3HP.js";import{Args as f,Flags as r}from"@oclif/core";import{readFileSync as g}from"node:fs";var s=class d extends c{static{m(this,"EvalCreate")}static description="Create test cases (evals) for an eval target";static examples=['<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',"<%= config.bin %> evals create <target-id> --from-file evals.json"];static args={targetId:f.string({description:"Eval target ID",required:!0})};static flags={name:r.string({char:"n",description:"Eval name"}),prompt:r.string({char:"p",description:"Test prompt"}),expectations:r.string({char:"e",description:"Assertion (repeatable)",multiple:!0}),"from-file":r.string({description:"Bulk create from JSON file"})};async run(){let{args:n,flags:e}=await this.parse(d);if(this.requireAuth(),e["from-file"]){let a=JSON.parse(g(e["from-file"],"utf-8")),p=Array.isArray(a)?a:a.evals;for(let t of p){let l=await i(`/api/eval-targets/${n.targetId}/evals`,{name:t.name,prompt:t.prompt,expectations:t.expectations??[],files:t.files??[]});this.log(`Created eval: ${l.name} (${l.id})`)}this.log(`Created ${p.length} evals.`);return}(!e.name||!e.prompt)&&this.error("Provide --name and --prompt, or use --from-file for bulk creation");let o=await i(`/api/eval-targets/${n.targetId}/evals`,{name:e.name,prompt:e.prompt,expectations:e.expectations??[]});this.log(`Created eval: ${o.name} (${o.id})`)}};export{s as default};

package/dist/commands/evals/optimize-description.js

@@ -1,4 +1,4 @@
-import{a as P,b as S}from"../../chunk-6QMIDH6X.js";import{f as b}from"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as N}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as y,f as q,g as w}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as E}from"../../chunk-ZLLER3HP.js";import{Args as Q,Flags as v}from"@oclif/core";var C=process.env.RULEMETRIC_WEB_URL??"https://rulemetric.com",_=30*6e4,R=class k extends N{static{E(this,"EvalOptimizeDescriptionCommand")}static description="Optimize eval target description for better triggering accuracy";static examples=["<%= config.bin %> evals optimize-description <target-id>","<%= config.bin %> evals optimize-description <target-id> --queries 10"];static args={targetId:Q.string({description:"Eval target ID",required:!0})};static flags={"max-iterations":v.integer({description:"Maximum optimization iterations",default:5}),model:v.string({description:"Model for trigger testing and improvement"}),queries:v.integer({description:"Number of trigger queries to generate",default:20}),"runs-per-query":v.integer({description:"Runs per query for reliability",default:3}),"poll-interval":v.integer({description:"Ms between approval polls",default:5e3}),timeout:v.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:i,flags:t}=await this.parse(k);this.requireAuth();let e=await y(`/api/eval-targets/${i.targetId}`);this.log(`Optimizing description for "${e.name}" v${e.version}`),this.log(`Current description: "${e.description}"`),this.log("Scores measure description-gated invocation choice (catalog decision protocol)."),this.log(""),this.log("Generating trigger queries...");let r=await this.generateTriggerQueries(e,t.queries,t),a=await q(`/api/eval-targets/${i.targetId}/trigger-evals`,{queries:r}),g=`${C}/eval-targets/${i.targetId}`;this.log(`
+import{a as P,b as S}from"../../chunk-BV3JMD3A.js";import{g as b}from"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as N}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as y,f as q,g as w}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as E}from"../../chunk-ZLLER3HP.js";import{Args as Q,Flags as v}from"@oclif/core";var C=process.env.RULEMETRIC_WEB_URL??"https://rulemetric.com",_=30*6e4,R=class k extends N{static{E(this,"EvalOptimizeDescriptionCommand")}static description="Optimize eval target description for better triggering accuracy";static examples=["<%= config.bin %> evals optimize-description <target-id>","<%= config.bin %> evals optimize-description <target-id> --queries 10"];static args={targetId:Q.string({description:"Eval target ID",required:!0})};static flags={"max-iterations":v.integer({description:"Maximum optimization iterations",default:5}),model:v.string({description:"Model for trigger testing and improvement"}),queries:v.integer({description:"Number of trigger queries to generate",default:20}),"runs-per-query":v.integer({description:"Runs per query for reliability",default:3}),"poll-interval":v.integer({description:"Ms between approval polls",default:5e3}),timeout:v.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:i,flags:t}=await this.parse(k);this.requireAuth();let e=await y(`/api/eval-targets/${i.targetId}`);this.log(`Optimizing description for "${e.name}" v${e.version}`),this.log(`Current description: "${e.description}"`),this.log("Scores measure description-gated invocation choice (catalog decision protocol)."),this.log(""),this.log("Generating trigger queries...");let r=await this.generateTriggerQueries(e,t.queries,t),a=await q(`/api/eval-targets/${i.targetId}/trigger-evals`,{queries:r}),g=`${C}/eval-targets/${i.targetId}`;this.log(`
 Generated ${r.length} trigger queries.`),this.log("Review and approve them at:"),this.log(`  ${g}`),this.log("  (Triggers tab \u2192 click to review)"),this.log(`
 Waiting for approval...`);let c=Date.now(),d=!1;for(;!d;){Date.now()-c>_&&this.error(`Timed out after ${_/6e4} minutes waiting for approval. Approve the trigger set at ${g} (Triggers tab) and re-run this command.`),await new Promise(h=>setTimeout(h,t["poll-interval"]));let p=(await y(`/api/eval-targets/${i.targetId}/trigger-evals?status=approved`)).find(h=>h.id===a.id);p&&(d=!0,r.length=0,r.push(...p.queries),this.log("Trigger queries approved!"))}let n=r.map((u,p)=>p).sort(()=>Math.random()-.5),s=Math.ceil(n.length*.6),o=n.slice(0,s),I=n.slice(s);await w(`/api/trigger-evals/${a.id}`,{trainIndices:o,testIndices:I});let x=o.map(u=>r[u]),D=I.map(u=>r[u]);this.log(`
 Split: ${x.length} train, ${D.length} test`);let m=e.description,T=m,$=0;for(let u=1;u<=t["max-iterations"];u++){this.log(`

package/dist/commands/evals/optimize.js

@@ -1,4 +1,4 @@
-import{b as P,c as C}from"../../chunk-BU3ZGLQG.js";import{a as V}from"../../chunk-PDXHRF4E.js";import{a as N}from"../../chunk-FG343JM3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-KFNFLMUR.js";import{f as F}from"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as A}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as w,f as m,g as S}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as k}from"../../chunk-ZLLER3HP.js";import{createInterface as _}from"node:readline/promises";import{Args as q,Flags as g}from"@oclif/core";var $=class O extends A{static{k(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:q.string({description:"Eval target ID",required:!0})};static flags={auto:g.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":g.integer({description:"Maximum optimization iterations",default:5}),threshold:g.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:g.string({description:"Model for eval execution"}),"analyzer-model":g.string({description:"Model for analysis/suggestions"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:n,flags:e}=await this.parse(O);this.requireAuth();let t=await w(`/api/eval-targets/${n.targetId}`),i=await w(`/api/eval-targets/${n.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let s=[...i].sort(()=>Math.random()-.5),c=Math.ceil(s.length*.6),u=s.slice(0,c),d=s.slice(c);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${u.length} train, ${d.length} test`);let h=parseFloat(e.threshold);this.log(`Config: max ${e["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),e.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let T=t.version,f=0,x=0;for(let l=1;l<=e["max-iterations"];l++){this.log(`
+import{b as P,c as C}from"../../chunk-BU3ZGLQG.js";import{a as V}from"../../chunk-TL57YUJV.js";import{a as N}from"../../chunk-U6AASXY3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-KFNFLMUR.js";import{g as F}from"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as A}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as w,f as m,g as S}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as k}from"../../chunk-ZLLER3HP.js";import{createInterface as _}from"node:readline/promises";import{Args as q,Flags as g}from"@oclif/core";var $=class O extends A{static{k(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:q.string({description:"Eval target ID",required:!0})};static flags={auto:g.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":g.integer({description:"Maximum optimization iterations",default:5}),threshold:g.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:g.string({description:"Model for eval execution"}),"analyzer-model":g.string({description:"Model for analysis/suggestions"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:n,flags:e}=await this.parse(O);this.requireAuth();let t=await w(`/api/eval-targets/${n.targetId}`),i=await w(`/api/eval-targets/${n.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let s=[...i].sort(()=>Math.random()-.5),c=Math.ceil(s.length*.6),u=s.slice(0,c),d=s.slice(c);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${u.length} train, ${d.length} test`);let h=parseFloat(e.threshold);this.log(`Config: max ${e["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),e.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let T=t.version,f=0,x=0;for(let l=1;l<=e["max-iterations"];l++){this.log(`
 === Iteration ${l} ===`),this.log("Running train evals...");let b=await this.runEvalsOnTarget(t,u,e),p=this.computePassRate(b);if(this.log(`Train pass rate: ${(p*100).toFixed(1)}%`),p>=h){this.log(`Train score ${(p*100).toFixed(1)}% >= threshold ${(h*100).toFixed(0)}%. Stopping.`);let o=await this.runEvalsOnTarget(t,d,e),z=this.computePassRate(o);this.log(`Final test score: ${(z*100).toFixed(1)}%`),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,testScore:z,analysis:{reason:"threshold_reached"},status:"completed"});break}this.log("Analyzing results...");let M=b.map(o=>({evalName:o.evalName,prompt:o.prompt,expectations:o.expectations,grading:o.grading})),a=await V({content:t.content,evalResults:M,model:e["analyzer-model"]??e.model,timeout:e.timeout});this.log(`
 Assessment: ${a.overallAssessment}`),this.log(`Suggestions: ${a.suggestions.length}`);for(let o of a.suggestions)this.log(`  - [${o.section}] ${o.reasoning}`);if(a.proposedContent===t.content){this.log("No changes proposed. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,analysis:{reason:"no_changes"},status:"completed"});break}if(!e.auto&&!await this.confirm(`
 Apply the proposed changes as a new version? [y/N] `)){this.log("Changes not applied. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,trainScore:p,analysis:{reason:"user_declined",suggestions:a.suggestions},status:"completed"});break}let r=await S(`/api/eval-targets/${t.id}`,{content:a.proposedContent});this.log(`Created v${r.version}`),this.log(`Running test evals on v${t.version} (baseline) and v${r.version}...`);let E=await this.runEvalsOnTarget(t,d,e),R=await this.runEvalsOnTarget(r,d,e),v=this.computePassRate(R),I=this.computePassRate(E);this.log(`Test pass rate: v${t.version} ${(I*100).toFixed(1)}% \u2192 v${r.version} ${(v*100).toFixed(1)}%`);let y=P(E,R);if(y&&this.log(C(y,t.version,r.version)),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:l,inputVersion:t.version,outputVersion:r.version,trainScore:p,testScore:v,analysis:{suggestions:a.suggestions,overallAssessment:a.overallAssessment,priorTestScore:I,verdict:y},status:"completed"}),v>f)f=v,T=r.version,x=0;else if(x++,x>=2){this.log("No improvement for 2 consecutive iterations. Stopping.");break}t=r}this.log(`

package/dist/commands/evals/refine-cases.js

@@ -1,4 +1,4 @@
-import{b as w,c as $}from"../../chunk-TSLBCZ5P.js";import"../../chunk-65LFPUDB.js";import{b as v}from"../../chunk-HBDKOU2N.js";import"../../chunk-KGU2IW54.js";import"../../chunk-M7QSC64D.js";import"../../chunk-ZKPURXUS.js";import"../../chunk-ISEUEYSC.js";import"../../chunk-GYKYJJNF.js";import"../../chunk-G7V4ZZZ3.js";import"../../chunk-LJE5J3EW.js";import"../../chunk-AHLVY52I.js";import"../../chunk-YXMAGC53.js";import"../../chunk-T7HZZ4DJ.js";import"../../chunk-DHIEGG4T.js";import"../../chunk-PPAHIVG5.js";import"../../chunk-5MXZJWQS.js";import"../../chunk-CT7FWSIU.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-FADINV4H.js";import"../../chunk-SWSOEYC7.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-FG343JM3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-KFNFLMUR.js";import"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as f}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as d,f as p,h}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import{Args as P,Flags as y}from"@oclif/core";var c=a(s=>s==null?"n/a":`${Math.round(s*100)}%`,"pct"),g=class s extends f{static{a(this,"EvalRefineCases")}static description="Retire non-discriminating eval cases and regenerate discriminating replacements (\xA76e)";static examples=["<%= config.bin %> evals refine-cases <target-id>            # preview","<%= config.bin %> evals refine-cases <target-id> --yes      # apply: retire + regenerate","<%= config.bin %> evals refine-cases <target-id> --yes --no-generate"];static args={targetId:P.string({description:"Eval target ID",required:!0})};static flags={yes:y.boolean({char:"y",description:"Apply (retire + regenerate). Without this, only previews."}),"no-generate":y.boolean({description:"Retire dead cases but do not regenerate replacements."})};async run(){let{args:b,flags:l}=await this.parse(s);this.requireAuth();let n=b.targetId,o=await d(`/api/eval-targets/${n}`),m=await d(`/api/eval-targets/${n}/judge-calibration`),i=m.caseDiscrimination,t=w(i);if(this.log(`Target "${o.name}" v${o.version} \u2014 judge ${m.judge.currentModel??"(unstamped)"}`),t.refused==="no-discrimination-data"){this.log("No case-discrimination data yet \u2014 run some evals with AND without the target first (`evals run` / the web Run Evals button), then refine.");return}if(this.log(`${i.total} diagnosable case(s): ${i.discriminating} discriminate, ${i.nonDiscriminating} flat/wrong-sign, ${i.unpaired} unpaired.`),t.refused==="all-discriminating"){this.log("Every case discriminates \u2014 nothing to retire. \u{1F389}");return}let C=i.cases.filter(e=>!e.discriminating),u=new Set(t.retire.map(e=>e.evalId));this.log(`
+import{b as w,c as $}from"../../chunk-FD53AP7T.js";import"../../chunk-DOEPIICK.js";import{b as v}from"../../chunk-VNGJRYBD.js";import"../../chunk-FLXWL4F4.js";import"../../chunk-ICLAHFKK.js";import"../../chunk-M7QSC64D.js";import"../../chunk-DZRBB34J.js";import"../../chunk-KR32Y5Q5.js";import"../../chunk-HBJMELKB.js";import"../../chunk-GYKYJJNF.js";import"../../chunk-G7V4ZZZ3.js";import"../../chunk-AHLVY52I.js";import"../../chunk-ULFSGFT2.js";import"../../chunk-55HFAX5Y.js";import"../../chunk-GWQ24FQP.js";import"../../chunk-PPAHIVG5.js";import"../../chunk-CT7FWSIU.js";import"../../chunk-5MXZJWQS.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-UGA2NFRG.js";import"../../chunk-VZYNZS5U.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-U6AASXY3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-KFNFLMUR.js";import"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as f}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as d,f as p,h}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import{Args as P,Flags as y}from"@oclif/core";var c=a(s=>s==null?"n/a":`${Math.round(s*100)}%`,"pct"),g=class s extends f{static{a(this,"EvalRefineCases")}static description="Retire non-discriminating eval cases and regenerate discriminating replacements (\xA76e)";static examples=["<%= config.bin %> evals refine-cases <target-id>            # preview","<%= config.bin %> evals refine-cases <target-id> --yes      # apply: retire + regenerate","<%= config.bin %> evals refine-cases <target-id> --yes --no-generate"];static args={targetId:P.string({description:"Eval target ID",required:!0})};static flags={yes:y.boolean({char:"y",description:"Apply (retire + regenerate). Without this, only previews."}),"no-generate":y.boolean({description:"Retire dead cases but do not regenerate replacements."})};async run(){let{args:b,flags:l}=await this.parse(s);this.requireAuth();let n=b.targetId,o=await d(`/api/eval-targets/${n}`),m=await d(`/api/eval-targets/${n}/judge-calibration`),i=m.caseDiscrimination,t=w(i);if(this.log(`Target "${o.name}" v${o.version} \u2014 judge ${m.judge.currentModel??"(unstamped)"}`),t.refused==="no-discrimination-data"){this.log("No case-discrimination data yet \u2014 run some evals with AND without the target first (`evals run` / the web Run Evals button), then refine.");return}if(this.log(`${i.total} diagnosable case(s): ${i.discriminating} discriminate, ${i.nonDiscriminating} flat/wrong-sign, ${i.unpaired} unpaired.`),t.refused==="all-discriminating"){this.log("Every case discriminates \u2014 nothing to retire. \u{1F389}");return}let C=i.cases.filter(e=>!e.discriminating),u=new Set(t.retire.map(e=>e.evalId));this.log(`
 Would retire ${t.retire.length} non-discriminating case(s):`);for(let e of C)u.has(e.evalId)&&this.log(`  \u2717 ${e.name} \u2014 ${c(e.withPassRate)} with vs ${c(e.withoutPassRate)} without (\u0394${(e.separation*100).toFixed(0)}pp) \u2014 ${e.note}`);for(let e of i.unpairedCases)u.has(e.evalId)&&this.log(`  \u2717 ${e.name} \u2014 ${c(e.withPassRate)} with vs ${c(e.withoutPassRate)} without \u2014 unpaired`);if(t.replacementsRequired&&this.log(`
 ${t.summary}. Replacements are generated and created BEFORE anything is retired \u2014 a target left under the floor drops out of the nightly evolution rotation with no error anywhere.`),!l.yes){this.log(`
 Preview only. Re-run with --yes to retire ${t.retire.length} case(s)${l["no-generate"]?"":" and generate replacements"}.`);return}let r=await $(t,{generate:a(e=>l["no-generate"]?Promise.resolve([]):v({content:o.content,name:o.name,feedback:e}),"generate"),createCase:a(e=>p(`/api/eval-targets/${n}/evals`,{name:e.name,prompt:e.prompt,expectations:e.expectations}),"createCase"),retireCase:a(async(e,R)=>{await h(`/api/eval-targets/${n}/evals/${e}`,{retire:!0,retiredReason:R})},"retireCase"),log:a(e=>this.log(e),"log")});if(r.skipped==="no-llm-backend"){this.warn(r.summary+" Install the `claude` CLI (or set ANTHROPIC_API_KEY), then run `evals refine-cases` again.");return}if(r.skipped==="no-replacements-generated"){this.warn(r.summary);return}this.log(`

package/dist/commands/evals/run.js

@@ -1,3 +1,3 @@
-import{d as k}from"../../chunk-FADINV4H.js";import"../../chunk-SWSOEYC7.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-FG343JM3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as $}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import{e as v,f as h}from"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as R}from"../../chunk-ZLLER3HP.js";import{Args as I,Flags as g}from"@oclif/core";var b=class _ extends ${static{R(this,"EvalRunCommand")}static description="Run evals against a target, optionally grade results, and post to API";static examples=["<%= config.bin %> evals run <target-id>","<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2","<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"];static args={targetId:I.string({description:"Eval target ID",required:!0})};static flags={parallel:g.integer({description:"Number of parallel eval executions",default:1}),"runs-per-eval":g.integer({description:"Number of runs per eval for variance analysis",default:1}),iteration:g.integer({description:"Iteration number for tracking",default:1}),"skip-baseline":g.boolean({description:"Skip baseline (without_target) runs",default:!1}),"skip-grading":g.boolean({description:"Skip LLM grading step",default:!1}),model:g.string({description:"Model for eval execution (default: user configured)"}),"grader-model":g.string({description:"Model for grading (default: same as execution model)"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:c,flags:r}=await this.parse(_);this.requireAuth();let i=await v(`/api/eval-targets/${c.targetId}`),l=await v(`/api/eval-targets/${c.targetId}/evals`);l.length===0&&this.error(`No evals found for target "${i.name}". Create some with \`rulemetric evals create\`.`),this.log(`Running ${l.length} eval(s) for "${i.name}" v${i.version}`),this.log(`Config: ${r["runs-per-eval"]} run(s) per eval, parallel=${r.parallel}, iteration=${r.iteration}`),r["skip-baseline"]&&this.log("Skipping baseline runs"),r["skip-grading"]&&this.log("Skipping grading"),this.log("");let m=[{name:"with_target",content:i.content}];r["skip-baseline"]||m.push({name:"without_target"});let p=[];for(let a of l)for(let n of m)for(let t=1;t<=r["runs-per-eval"];t++)p.push({eval_:a,configuration:n.name,targetContent:n.content,runNumber:t});let u=[],y=r.parallel;for(let a=0;a<p.length;a+=y){let n=p.slice(a,a+y),t=await Promise.all(n.map(d=>this.executeAndGrade(d,i,r)));u.push(...t)}this.log(""),this.log("Posting results to API...");for(let a of u)await h("/api/eval-runs",a);try{await h(`/api/eval-targets/${c.targetId}/label-state`,{})}catch{}this.log(""),this.log("=== Results Summary ===");for(let a of m){let n=u.filter(e=>e.configuration===a.name),t=n.filter(e=>e.status==="completed"),d=t.filter(e=>Object.keys(e.grading).length>0);if(this.log(`
+import{d as k}from"../../chunk-UGA2NFRG.js";import"../../chunk-VZYNZS5U.js";import"../../chunk-U2NT5OSQ.js";import"../../chunk-KGURGIZ2.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import"../../chunk-U6AASXY3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as $}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import{e as v,f as h}from"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as R}from"../../chunk-ZLLER3HP.js";import{Args as I,Flags as g}from"@oclif/core";var b=class _ extends ${static{R(this,"EvalRunCommand")}static description="Run evals against a target, optionally grade results, and post to API";static examples=["<%= config.bin %> evals run <target-id>","<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2","<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"];static args={targetId:I.string({description:"Eval target ID",required:!0})};static flags={parallel:g.integer({description:"Number of parallel eval executions",default:1}),"runs-per-eval":g.integer({description:"Number of runs per eval for variance analysis",default:1}),iteration:g.integer({description:"Iteration number for tracking",default:1}),"skip-baseline":g.boolean({description:"Skip baseline (without_target) runs",default:!1}),"skip-grading":g.boolean({description:"Skip LLM grading step",default:!1}),model:g.string({description:"Model for eval execution (default: user configured)"}),"grader-model":g.string({description:"Model for grading (default: same as execution model)"}),timeout:g.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:c,flags:r}=await this.parse(_);this.requireAuth();let i=await v(`/api/eval-targets/${c.targetId}`),l=await v(`/api/eval-targets/${c.targetId}/evals`);l.length===0&&this.error(`No evals found for target "${i.name}". Create some with \`rulemetric evals create\`.`),this.log(`Running ${l.length} eval(s) for "${i.name}" v${i.version}`),this.log(`Config: ${r["runs-per-eval"]} run(s) per eval, parallel=${r.parallel}, iteration=${r.iteration}`),r["skip-baseline"]&&this.log("Skipping baseline runs"),r["skip-grading"]&&this.log("Skipping grading"),this.log("");let m=[{name:"with_target",content:i.content}];r["skip-baseline"]||m.push({name:"without_target"});let p=[];for(let a of l)for(let n of m)for(let t=1;t<=r["runs-per-eval"];t++)p.push({eval_:a,configuration:n.name,targetContent:n.content,runNumber:t});let u=[],y=r.parallel;for(let a=0;a<p.length;a+=y){let n=p.slice(a,a+y),t=await Promise.all(n.map(d=>this.executeAndGrade(d,i,r)));u.push(...t)}this.log(""),this.log("Posting results to API...");for(let a of u)await h("/api/eval-runs",a);try{await h(`/api/eval-targets/${c.targetId}/label-state`,{})}catch{}this.log(""),this.log("=== Results Summary ===");for(let a of m){let n=u.filter(e=>e.configuration===a.name),t=n.filter(e=>e.status==="completed"),d=t.filter(e=>Object.keys(e.grading).length>0);if(this.log(`
 ${a.name}:`),this.log(`  Runs: ${t.length}/${n.length} completed`),d.length>0){let e=d.map(o=>o.grading.summary?.pass_rate??0),s=e.reduce((o,f)=>o+f,0)/e.length;this.log(`  Avg pass_rate: ${(s*100).toFixed(1)}%`)}for(let e of n){let s=l.find(x=>x.id===e.evalId)?.name??e.evalId,o=e.grading.summary?.pass_rate,f=o!==void 0?` pass_rate=${(o*100).toFixed(0)}%`:"",w=e.timing.duration_ms?` ${(e.timing.duration_ms/1e3).toFixed(1)}s`:"";this.log(`  [${e.status}] ${s} run#${e.runNumber}${f}${w}`)}}if(!r["skip-baseline"]&&u.length>0){let a=u.filter(t=>t.configuration==="with_target"&&Object.keys(t.grading).length>0),n=u.filter(t=>t.configuration==="without_target"&&Object.keys(t.grading).length>0);if(a.length>0&&n.length>0){let t=a.map(s=>s.grading.summary?.pass_rate??0).reduce((s,o)=>s+o,0)/a.length,d=n.map(s=>s.grading.summary?.pass_rate??0).reduce((s,o)=>s+o,0)/n.length,e=t-d;this.log(`
 Delta: ${e>0?"+":""}${(e*100).toFixed(1)}% (with_target vs without_target)`)}}}async executeAndGrade(c,r,i){return k(c,{targetId:r.id,targetVersion:r.version,iteration:i.iteration,timeout:i.timeout,model:i.model,graderModel:i["grader-model"],skipGrading:i["skip-grading"],targetType:r.type,targetName:r.name},l=>this.log(l))}};export{b as default};

package/dist/commands/evals/validate-judge.js

@@ -1 +1 @@
-import{c,d as m}from"../../chunk-3H44B6PB.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-FG343JM3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-4GOXEEXZ.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-JNLDISBA.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as u}from"../../chunk-WUCACPGC.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-WL5ZWQAS.js";import"../../chunk-KEVDLLHT.js";import"../../chunk-DX6IPDED.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-GIXHG5FK.js";import"../../chunk-X6CF5KTW.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as r}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as s}from"@oclif/core";var o=r(n=>n==null?"\u2014":`${(n*100).toFixed(0)}%`,"pct"),g=class n extends u{static{r(this,"EvalValidateJudge")}static description="Score a second model against your human judge-review labels (\xA76g validation pass)";static examples=["<%= config.bin %> evals validate-judge <target-id>                       # auto-detect installed CLIs","<%= config.bin %> evals validate-judge <target-id> --engine codex --model gpt-5.5","<%= config.bin %> evals validate-judge <target-id> --model claude-sonnet-5 --json","<%= config.bin %> evals validate-judge <target-id> --limit 5             # quick sample"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={model:s.string({description:"Second reviewer model. Default: resolved from your installed tooling (codex: your ~/.codex/config.toml model; claude: a tier that differs from the original judge)."}),engine:s.string({description:"CLI that runs the second reviewer. `auto` prefers your installed codex for a cross-vendor reviewer \u2014 two Claude models share a training lineage, so their agreement overstates independence.",options:["auto","claude","codex"],default:"auto"}),limit:s.integer({description:"Sample only the N most-recently-annotated runs (cost control / smoke)"}),"skip-reviewed":s.boolean({description:"Only re-grade assertions this reviewer model has not ruled on yet",default:!1}),parallel:s.integer({description:"Concurrent re-grades",default:3}),timeout:s.integer({description:"Timeout per re-grade in ms",default:12e4}),json:s.boolean({description:"Emit the full report as JSON"})};async run(){let{args:h,flags:i}=await this.parse(n);this.requireAuth();let a;try{a=await m({targetId:h.targetId,engineFlag:i.engine,modelFlag:i.model,limit:i.limit,parallel:i.parallel,timeoutMs:i.timeout,skipReviewed:i["skip-reviewed"],log:r(t=>this.log(t),"log"),warn:r(t=>this.warn(t),"warn")})}catch(t){throw t instanceof c&&this.error(t.message),t}let{report:e,judgeModels:l,reviewerModel:d}=a;if(l.length===1&&l[0]===d&&this.warn(`Second reviewer resolved to the SAME model as the original judge (${l[0]}) \u2014 this measures self-consistency, not independent review.`),i.json){this.log(JSON.stringify({target:a.targetName,judgeA:l,judgeB:d,persisted:a.persisted,alreadyReviewed:a.alreadyReviewed,...e},null,2));return}if(e.pairs===0&&a.alreadyReviewed>0){this.log(`All ${a.alreadyReviewed} labeled assertion(s) already reviewed by ${d} \u2014 nothing new to grade.`);return}if(this.log(""),this.log(`Second reviewer: ${d} \u2014 scored against ${e.pairs} human-labeled assertion(s)`+(e.skipped?` (${e.skipped} skipped)`:"")),this.log(""),this.log(`  reviewer vs HUMAN:   ${o(e.bVsHuman.overall.rate)} overall \xB7 ${o(e.bVsHuman.onJudgePass.rate)} where judge passed \xB7 ${o(e.bVsHuman.onJudgeFail.rate)} where judge failed`),this.log(`  original judge vs human: ${o(e.aVsHuman.overall.rate)} (your approve rate)`),this.log(`  reviewer vs judge:   ${o(e.bVsA.overall.rate)} (inter-model, not truth)`),this.log(`  Cohen's \u03BA (reviewer vs human): ${e.kappaBvsHuman==null?"undefined (degenerate marginals)":e.kappaBvsHuman.toFixed(2)}`),this.log(""),e.suggestedAnchorWeight!=null&&this.log(`  suggested \xA76g anchor weight if used as a passive label source: ${e.suggestedAnchorWeight.toFixed(2)} (misclassification attenuation (2\xB7acc\u22121)\xB2; validated:false either way)`),e.disagreements.length>0){this.log(""),this.log(`  ${e.disagreements.length} disagreement(s) with your labels \u2014 flagged on the Review tab:`);for(let t of e.disagreements.slice(0,10))this.log(`    run ${t.runId.slice(0,8)} #${t.expectationIndex}: human=${t.humanPassed?"pass":"fail"} reviewer=${t.judgeBPassed?"pass":"fail"} \u2014 ${t.text.slice(0,80)}`);e.disagreements.length>10&&this.log(`    \u2026 and ${e.disagreements.length-10} more (use --json for all)`)}}};export{g as default};
+import{c,d as m}from"../../chunk-YT25HLWB.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-U6AASXY3.js";import"../../chunk-CDY7TQFZ.js";import"../../chunk-LARY6ZFT.js";import"../../chunk-JP6ULOEX.js";import"../../chunk-SVZYRGO5.js";import"../../chunk-WULEBQPI.js";import"../../chunk-GPBN4ROE.js";import"../../chunk-6XTLP2G7.js";import"../../chunk-PJ3WHQ67.js";import{a as u}from"../../chunk-TQFHRD5B.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-5LTJNYCP.js";import"../../chunk-3EZZGA72.js";import"../../chunk-XAGNGQL3.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-5VWVQCNR.js";import"../../chunk-MHLHLJ5T.js";import"../../chunk-XQYAVSUQ.js";import"../../chunk-PZNWH4VQ.js";import{a as r}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as s}from"@oclif/core";var o=r(n=>n==null?"\u2014":`${(n*100).toFixed(0)}%`,"pct"),g=class n extends u{static{r(this,"EvalValidateJudge")}static description="Score a second model against your human judge-review labels (\xA76g validation pass)";static examples=["<%= config.bin %> evals validate-judge <target-id>                       # auto-detect installed CLIs","<%= config.bin %> evals validate-judge <target-id> --engine codex --model gpt-5.5","<%= config.bin %> evals validate-judge <target-id> --model claude-sonnet-5 --json","<%= config.bin %> evals validate-judge <target-id> --limit 5             # quick sample"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={model:s.string({description:"Second reviewer model. Default: resolved from your installed tooling (codex: your ~/.codex/config.toml model; claude: a tier that differs from the original judge)."}),engine:s.string({description:"CLI that runs the second reviewer. `auto` prefers your installed codex for a cross-vendor reviewer \u2014 two Claude models share a training lineage, so their agreement overstates independence.",options:["auto","claude","codex"],default:"auto"}),limit:s.integer({description:"Sample only the N most-recently-annotated runs (cost control / smoke)"}),"skip-reviewed":s.boolean({description:"Only re-grade assertions this reviewer model has not ruled on yet",default:!1}),parallel:s.integer({description:"Concurrent re-grades",default:3}),timeout:s.integer({description:"Timeout per re-grade in ms",default:12e4}),json:s.boolean({description:"Emit the full report as JSON"})};async run(){let{args:h,flags:i}=await this.parse(n);this.requireAuth();let a;try{a=await m({targetId:h.targetId,engineFlag:i.engine,modelFlag:i.model,limit:i.limit,parallel:i.parallel,timeoutMs:i.timeout,skipReviewed:i["skip-reviewed"],log:r(t=>this.log(t),"log"),warn:r(t=>this.warn(t),"warn")})}catch(t){throw t instanceof c&&this.error(t.message),t}let{report:e,judgeModels:l,reviewerModel:d}=a;if(l.length===1&&l[0]===d&&this.warn(`Second reviewer resolved to the SAME model as the original judge (${l[0]}) \u2014 this measures self-consistency, not independent review.`),i.json){this.log(JSON.stringify({target:a.targetName,judgeA:l,judgeB:d,persisted:a.persisted,alreadyReviewed:a.alreadyReviewed,...e},null,2));return}if(e.pairs===0&&a.alreadyReviewed>0){this.log(`All ${a.alreadyReviewed} labeled assertion(s) already reviewed by ${d} \u2014 nothing new to grade.`);return}if(this.log(""),this.log(`Second reviewer: ${d} \u2014 scored against ${e.pairs} human-labeled assertion(s)`+(e.skipped?` (${e.skipped} skipped)`:"")),this.log(""),this.log(`  reviewer vs HUMAN:   ${o(e.bVsHuman.overall.rate)} overall \xB7 ${o(e.bVsHuman.onJudgePass.rate)} where judge passed \xB7 ${o(e.bVsHuman.onJudgeFail.rate)} where judge failed`),this.log(`  original judge vs human: ${o(e.aVsHuman.overall.rate)} (your approve rate)`),this.log(`  reviewer vs judge:   ${o(e.bVsA.overall.rate)} (inter-model, not truth)`),this.log(`  Cohen's \u03BA (reviewer vs human): ${e.kappaBvsHuman==null?"undefined (degenerate marginals)":e.kappaBvsHuman.toFixed(2)}`),this.log(""),e.suggestedAnchorWeight!=null&&this.log(`  suggested \xA76g anchor weight if used as a passive label source: ${e.suggestedAnchorWeight.toFixed(2)} (misclassification attenuation (2\xB7acc\u22121)\xB2; validated:false either way)`),e.disagreements.length>0){this.log(""),this.log(`  ${e.disagreements.length} disagreement(s) with your labels \u2014 flagged on the Review tab:`);for(let t of e.disagreements.slice(0,10))this.log(`    run ${t.runId.slice(0,8)} #${t.expectationIndex}: human=${t.humanPassed?"pass":"fail"} reviewer=${t.judgeBPassed?"pass":"fail"} \u2014 ${t.text.slice(0,80)}`);e.disagreements.length>10&&this.log(`    \u2026 and ${e.disagreements.length-10} more (use --json for all)`)}}};export{g as default};