@rulemetric/cli 0.14.0 → 0.14.2

package/dist/chunk-PUX6P5KC.js

@@ -1,2 +0,0 @@
-import{a as w}from"./chunk-6XTLP2G7.js";import{a as l}from"./chunk-ZLLER3HP.js";import{spawn as y}from"node:child_process";function b(o){let i=["--print","--mode","json","--no-session","--no-tools","--no-extensions","--no-skills","--no-prompt-templates"];return o.systemPrompt&&i.push("--system-prompt",o.systemPrompt),o.model&&i.push("--model",o.model),i.push(o.prompt),i}l(b,"buildPiArgs");var v=l((o,i)=>`pi:${o}/${i}`,"piModelId");function c(o){return typeof o=="number"&&Number.isFinite(o)?o:0}l(c,"finiteNumber");function T(o){let i=null;for(let r of o.split(`
-`)){if(!r.trim())continue;let s;try{s=JSON.parse(r)}catch{continue}let p=s;p.type==="agent_end"&&Array.isArray(p.messages)&&(i=p.messages)}if(!i)return{ok:!1,reason:"pi exited without an agent_end event in its JSON stream"};let m=i.filter(r=>r.role==="assistant"),e=m[m.length-1];if(!e)return{ok:!1,reason:"pi agent_end carried no assistant message"};if(e.stopReason==="error"||e.stopReason==="aborted")return{ok:!1,reason:`pi provider error: ${String(e.errorMessage??e.stopReason).slice(0,500)}`};let g=(Array.isArray(e.content)?e.content:[]).map(r=>{if(!r||typeof r!="object")return"";let s=r;return s.type==="text"&&typeof s.text=="string"?s.text:""}).join("");if(!g.trim())return{ok:!1,reason:`pi assistant message held no text (stopReason ${String(e.stopReason)})`};let k=typeof e.provider=="string"?e.provider:void 0,f=typeof e.model=="string"?e.model:void 0,u=e.usage&&typeof e.usage=="object"?e.usage:null,d,a;if(u){let r=c(u.input),s=c(u.output),p=c(u.cacheRead),t=c(u.cacheWrite),n=c(u.totalTokens);(r||s||p||t||n)&&(d={inputTokens:r,outputTokens:s,cacheReadTokens:p,cacheWriteTokens:t,totalTokens:n||r+s+p+t});let P=u.cost;if(P&&typeof P=="object"){let x=P.total;typeof x=="number"&&Number.isFinite(x)&&(a=x)}}return{ok:!0,output:g,provider:k,model:f,usage:d,costUsd:a}}l(T,"parsePiAgentEnd");async function E(o){let{timeout:i=12e4,workDir:m=process.cwd()}=o,e=w("pi");if(!e)return{output:"",exitCode:null,durationMs:0,error:"Pi CLI not found on PATH, well-known install dirs, or the node bin dir \u2014 is it installed?"};let g=Date.now();return new Promise(k=>{let f={...process.env};delete f.HTTPS_PROXY;let u=y(e,b(o),{cwd:m,env:f,stdio:["ignore","pipe","pipe"]}),d="",a="",r=!1,s=l(t=>{r||(r=!0,k({...t,durationMs:Date.now()-g}))},"finish");u.stdout.on("data",t=>{d+=t.toString(),d.length>1024*1024&&(d=d.slice(-1024*1024))}),u.stderr.on("data",t=>{a+=t.toString(),a.length>64*1024&&(a=a.slice(-64*1024))});let p=setTimeout(()=>{u.kill("SIGTERM"),s({output:"",exitCode:null,error:`pi timed out after ${i}ms`})},i);u.on("error",t=>{clearTimeout(p),s({output:"",exitCode:null,error:`failed to spawn pi: ${t.message}`})}),u.on("close",t=>{clearTimeout(p);let n=T(d);if(!n.ok){s({output:"",exitCode:t,error:`${n.reason}${a.trim()?`: ${a.trim().slice(-500)}`:""}`});return}if(t!==0){s({output:n.output,exitCode:t,usage:n.usage,costUsd:n.costUsd,error:`pi failed (exit ${t}): ${a.trim().slice(-500)||"no stderr"}`});return}s({output:n.output,exitCode:t,provider:n.provider,model:n.provider&&n.model?v(n.provider,n.model):n.model,usage:n.usage,costUsd:n.costUsd})})})}l(E,"executePiEval");export{b as a,v as b,T as c,E as d};

package/dist/chunk-SBGXIPRN.js

@@ -1,2 +0,0 @@
-import{a as r}from"./chunk-6XTLP2G7.js";import{a as l}from"./chunk-ZLLER3HP.js";import{readFileSync as x}from"node:fs";import{homedir as m}from"node:os";import{join as a}from"node:path";function p(i){for(let n of i.split(`
-`)){let o=n.trim();if(o.startsWith("["))break;let e=/^model\s*=\s*"([^"]+)"/.exec(o);if(e)return e[1]}return null}l(p,"parseCodexConfigModel");function B(i){return/requires a newer version of Codex|is not supported when using Codex/i.test(i)}l(B,"isCodexModelUnsupported");var u="gpt-5.5";function M(i=a(m(),".codex")){let n=null;try{n=p(x(a(i,"config.toml"),"utf8"))}catch{}return{codexBin:r("codex"),claudeBin:r("claude"),codexConfiguredModel:n}}l(M,"detectEngines");function $(i){let{engineFlag:n,modelFlag:o,availability:e,judgeModels:d}=i,s=o??e.codexConfiguredModel??u,g=o?`--model ${o}`:e.codexConfiguredModel?`model ${e.codexConfiguredModel} from your ~/.codex/config.toml`:`fallback model ${u} (no ~/.codex/config.toml model found)`,t=o??(d.some(f=>/opus/i.test(f))?"sonnet":null),c=o?`--model ${o}`:t?`model ${t} (differs from the ${d.join(", ")} judge)`:"CLI default model";if(n==="codex"||n==="auto"&&e.codexBin){if(!e.codexBin)throw new Error("Codex CLI not found on PATH, well-known install dirs, or the node bin dir \u2014 install it or use --engine claude.");return{engine:"codex",model:s,rationale:`codex at ${e.codexBin} (cross-vendor reviewer), ${g}`}}if(!e.claudeBin)throw new Error("No reviewer CLI found \u2014 neither codex nor claude is installed on this machine.");return{engine:"claude",model:t,rationale:n==="auto"?`claude at ${e.claudeBin} (codex not installed \u2014 same-vendor reviewer, agreement will overstate independence), ${c}`:`claude at ${e.claudeBin}, ${c}`}}l($,"chooseReviewer");export{p as a,B as b,u as c,M as d,$ as e};

package/dist/chunk-SJKS3PYX.js

@@ -1,109 +0,0 @@
-import{a}from"./chunk-ZLLER3HP.js";function r(o){let s=o.dismissed.filter(e=>e.label.trim()!==""),n=o.repeated.filter(e=>e.label.trim()!==""),i=(o.installed??[]).filter(e=>e.trim()!=="");if(s.length===0&&n.length===0&&i.length===0)return"";let t=[];return(s.length>0||n.length>0)&&t.push("","## ADVICE YOU HAVE ALREADY GIVEN THIS USER","This is your own prior output for this project. Use it to say something NEW."),s.length>0&&t.push("","DISMISSED by the user \u2014 do NOT propose these again, and do not propose a","reworded version of them. A paraphrase of a dismissed suggestion is still","the dismissed suggestion:",...s.map(e=>`- [${e.category}] ${e.label}`)),n.length>0&&t.push("","ALREADY GENERATED, sometimes repeatedly. Repeat one ONLY if it is still","the single most important finding in the data \u2014 otherwise prefer a new","observation over restating these:",...n.map(e=>`- [${e.category}] ${e.label} (generated ${e.timesGenerated}x)`)),i.length>0&&(t.push("","## ALREADY INSTALLED IN THIS PROJECT'S INSTRUCTION FILES","The agent reads these on every session in this project. They are the","user's own rules, not your suggestions. Do not propose them, and do not","propose a reworded version of one.","","If an idea is PARTLY here, propose only the part that is not \u2014 a block","mixing new asks with restatements is refused whole and its new asks are","lost with it. One ask per proposal is always safe.","",...i.map(e=>`- ${e}`)),(o.installedOmitted??0)>0&&t.push(`- (+${o.installedOmitted} more not shown \u2014 this project has more rules than fit here,`,"  so treat the list as a sample rather than the full set.)")),t.join(`
-`)}a(r,"renderGeneratorMemory");var d=[{name:"project_areas",usesGeneratorMemory:!0,prompt:`Analyze this Claude Code usage data and identify project areas.
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "areas": [
-    {"name": "Area name", "session_count": N, "description": "2-3 sentences about what was worked on and how Claude Code was used."}
-  ]
-}
-Include 4-5 areas. Skip internal CC operations.`,maxTokens:8192},{name:"interaction_style",prompt:`Analyze this Claude Code usage data and describe the user's interaction style.
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "narrative": "2-3 paragraphs analyzing HOW the user interacts with Claude Code. Use second person 'you'. Describe patterns: iterate quickly vs detailed upfront specs? Interrupt often or let Claude run? Include specific examples. Use **bold** for key insights.",
-  "key_pattern": "One sentence summary of most distinctive interaction style"
-}`,maxTokens:8192},{name:"what_works",usesGeneratorMemory:!0,prompt:`Analyze this Claude Code usage data and identify what's working well for this user. Use second person ("you").
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "intro": "1 sentence of context",
-  "impressive_workflows": [
-    {"title": "Short title (3-6 words)", "description": "2-3 sentences describing the impressive workflow or approach. Use 'you' not 'the user'."}
-  ]
-}
-Include 3 impressive workflows.`,maxTokens:8192},{name:"friction_analysis",usesGeneratorMemory:!0,prompt:`Analyze this Claude Code usage data and identify friction points for this user. Use second person ("you").
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "intro": "1 sentence summarizing friction patterns",
-  "categories": [
-    {"category": "Concrete category name", "description": "1-2 sentences explaining this category and what could be done differently. Use 'you' not 'the user'.", "examples": ["Specific example with consequence", "Another example"]}
-  ]
-}
-Include 3 friction categories with 2 examples each.`,maxTokens:8192},{name:"suggestions",usesGeneratorMemory:!0,timeoutMs:42e4,prompt:`Analyze this cross-harness coding-agent usage data and suggest the smallest changes that address the evidence.
-
-Choose the artifact honestly:
-- "instruction": durable behavioral guidance for CLAUDE.md / AGENTS.md.
-- "skill": a reusable, user-invoked markdown workflow.
-- "hook": lifecycle enforcement that runs a command.
-- "script": a repeated command chain worth making deterministic.
-- "mcp": an external tool, database, or API integration.
-- "subagent": a delegated specialist or multi-agent workflow.
-- "routine": periodic work or a scheduled check.
-- "feature": human-facing advice that does not fit the types above.
-
-Executable artifacts (hook, script, mcp, subagent, routine) are proposals for human review, never silent installs. Do not translate Codex evidence into Claude-only commands.
-Write each proposal ONCE, as one candidate. Do not repeat a proposal in another form.
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "recommendation_candidates": [
-    {
-      "title": "Short stable title",
-      "artifact": "instruction|skill|hook|script|mcp|subagent|routine|feature",
-      "canonicalBody": "Harness-neutral meaning, or the complete proposed artifact",
-      "rationale": "Why the evidence supports this change",
-      "evidence": [{"sessionId": "optional session id", "finding": "Observed behavior"}],
-      "outcomeMetric": "The outcome or recurrence metric this change should improve"
-    }
-  ],
-  "usage_patterns": [
-    {"title": "Short title", "suggestion": "1-2 sentence summary", "detail": "3-4 sentences explaining how this applies", "copyable_prompt": "A specific prompt to copy and try"}
-  ]
-}
-Include 3-6 candidates. Per-harness renderings are derived from canonicalBody \u2014 do not write them.`,maxTokens:8192},{name:"on_the_horizon",usesGeneratorMemory:!0,timeoutMs:24e4,prompt:`Analyze this Claude Code usage data and identify future opportunities.
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "intro": "1 sentence about evolving AI-assisted development",
-  "opportunities": [
-    {"title": "Short title (4-8 words)", "whats_possible": "2-3 ambitious sentences about autonomous workflows", "how_to_try": "1-2 sentences mentioning relevant tooling", "copyable_prompt": "Detailed prompt to try"}
-  ]
-}
-Include 3 opportunities. Think BIG - autonomous workflows, parallel agents, iterating against tests.`,maxTokens:8192},{name:"fun_ending",prompt:`Analyze this Claude Code usage data and find a memorable moment.
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "headline": "A memorable QUALITATIVE moment from the transcripts - not a statistic. Something human, funny, or surprising.",
-  "detail": "Brief context about when/where this happened"
-}
-Find something genuinely interesting or amusing from the session summaries.`,maxTokens:8192}],l=`You're writing an "At a Glance" summary for a Claude Code usage insights report.
-Use this 4-part structure:
-1. **What's working** - What is the user's unique style and what impactful things have they done? Don't be fluffy.
-2. **What's hindering you** - Split into (a) Claude's fault and (b) user-side friction. Be honest but constructive.
-3. **Quick wins to try** - Specific Claude Code features they could try, or a compelling workflow technique.
-4. **Ambitious workflows for better models** - As models improve, what workflows should they prepare for?
-Keep each section to 2-3 sentences. Use a coaching tone.
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "whats_working": "(refer to instructions above)",
-  "whats_hindering": "(refer to instructions above)",
-  "quick_wins": "(refer to instructions above)",
-  "ambitious_workflows": "(refer to instructions above)"
-}`,c=`Analyze this coding session and produce a structured analysis.
-The session is from a non-Claude-Code tool, so analyze the conversation and tool usage patterns.
-
-RESPOND WITH ONLY A VALID JSON OBJECT:
-{
-  "underlying_goal": "1-2 sentence description of what the user was trying to accomplish",
-  "goal_categories": {"category": count},
-  "outcome": "achieved|mostly_achieved|partially_achieved|not_achieved|abandoned",
-  "session_type": "quick_task|iterative_refinement|exploration|debugging|greenfield",
-  "friction": {"wrong_approach": 0, "tool_failure": 0, "excessive_changes": 0, "misunderstanding": 0},
-  "friction_detail": "Brief description of any friction encountered, or empty string",
-  "primary_success": "Brief description of main achievement, or empty string",
-  "brief_summary": "2-3 sentence summary of the session"
-}
-
-Goal categories to use: debugging, feature_development, refactoring, testing, documentation, investigation, exploration, code_review, deployment, configuration, end_to_end_testing, logging_documentation, ui_testing.
-
-Outcome guide:
-- achieved: Task completed successfully
-- mostly_achieved: Main goal done but minor items remain
-- partially_achieved: Some progress but significant work left
-- not_achieved: Attempted but failed
-- abandoned: User gave up or switched tasks`;export{r as a,d as b,l as c,c as d};

package/dist/chunk-SPJ73X7A.js

@@ -1,4 +0,0 @@
-import{d as I}from"./chunk-E7GQJML3.js";import{a as O}from"./chunk-YNLHJHUP.js";import{a as J}from"./chunk-6XTLP2G7.js";import{y as T}from"./chunk-TWPG3TEQ.js";import{a as L}from"./chunk-74QIUVWY.js";import{a as t}from"./chunk-ZLLER3HP.js";import{spawn as z}from"node:child_process";import{existsSync as V,mkdirSync as Z,readdirSync as K}from"node:fs";import{homedir as q,tmpdir as G}from"node:os";import{dirname as W,join as d,resolve as F}from"node:path";var Q=10800*1e3,X=/^rm-[a-z0-9-]+-task$/,Y=/^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/,y=/^[A-Za-z0-9._-]+$/;function rr(r,o,n){return new Promise((l,i)=>{let a=z(r,o,{cwd:n.cwd,env:n.env,stdio:["ignore","pipe","pipe"]}),e="",g="",m="",_=setTimeout(()=>a.kill("SIGKILL"),n.timeoutMs);a.stdout.on("data",s=>{let w=s.toString("utf8");if(e+=w,!n.onLine)return;m+=w;let $=m.split(`
-`);m=$.pop()??"";for(let p of $)n.onLine(p)}),a.stderr.on("data",s=>{g+=s.toString("utf8")}),a.on("error",s=>{clearTimeout(_),i(s)}),a.on("close",s=>{clearTimeout(_),m&&n.onLine&&n.onLine(m),l({status:s,stdout:e,stderr:g})})})}t(rr,"defaultRun");var nr={harnessDir:t(()=>O(),"harnessDir"),resolveBin:J,run:rr,exists:V,mkdir:t(r=>Z(r,{recursive:!0}),"mkdir"),readdir:K,jobsRoot:d(G(),"rulemetric-harbor"),now:t(()=>Date.now(),"now")};function er(r){if(r.trials&&r.trials.length>0){let o=r.trials.filter(n=>n.reward!=null);return{n:o.length,passes:o.filter(n=>n.reward>=.5).length}}return r.reward_mean==null||!r.n_trials?{n:r.n_trials??0,passes:null}:{n:r.n_trials,passes:Math.round(r.reward_mean*r.n_trials)}}t(er,"armReward");function sr(r){if(!X.test(r.task??""))throw new Error(`harbor_ab: refusing task "${r.task}" \u2014 not an rm-*-task name; a task is a directory name, never a path`);if(!Y.test(r.instructions??"")||r.instructions.split("/").includes(".."))throw new Error(`harbor_ab: refusing instructions "${r.instructions}" \u2014 must be a plain relative path under scripts/harbor`);if(!y.test(r.model??""))throw new Error(`harbor_ab: refusing model "${r.model}"`);if(!Number.isInteger(r.nTrials)||r.nTrials<1||r.nTrials>200)throw new Error(`harbor_ab: refusing nTrials=${r.nTrials} \u2014 must be an integer in 1..200`);if(r.class!=="proven"&&r.class!=="negative"&&r.class!=="unvalidated")throw new Error(`harbor_ab: unknown task class "${r.class}"`);if(r.axis!=="reward"&&r.axis!=="cost")throw new Error(`harbor_ab: unknown axis "${r.axis}"`)}t(sr,"validate");function tr(r,o){let n=new Set((process.env.PATH??"").split(":").filter(Boolean));for(let l of["harbor","docker","claude","codex","python3"]){let i=r.resolveBin(l);i&&n.add(W(i))}for(let l of[d(q(),".local","bin"),"/opt/homebrew/bin","/usr/local/bin"])n.add(l);return{...process.env,PATH:[...n].join(":"),HARBOR_JOBS_DIR:o}}t(tr,"spawnEnv");function f(r,o=20){return r.trim().split(`
-`).slice(-o).join(`
-`)}t(f,"tail");async function j(r,o,n,l,i,a=[]){let e=await r.run("python3",[d(o,l),...i],{cwd:o,env:n,timeoutMs:3e5});if(e.status!==0&&!a.includes(e.status))return{status:e.status,json:null,stderr:e.stderr};try{return{status:e.status,json:JSON.parse(e.stdout),stderr:e.stderr}}catch{throw new Error(`harbor_ab: ${l} printed something that is not JSON: ${f(e.stdout,5)}`)}}t(j,"readJson");async function wr(r,o,n=nr){let l=t(u=>o?.logger?.info?.(u)??console.log(u),"log"),i=r.measurement?L.parse(r.measurement):null,a=i?.engine??"claude";if(i){if(r={...r,model:i.model??T,class:a==="claude"&&i.model===T?r.class:"unvalidated"},a==="codex"&&r.axis==="cost")throw new Error("harbor_ab: dollar-cost telemetry is unavailable for Codex");if(!n.resolveBin(a))throw new Error(`harbor_ab: ${a} engine unavailable on this worker`)}sr(r);let e=n.harnessDir();if(!e)throw new Error("harbor_ab: no Harbor harness on this worker \u2014 scripts/harbor/run.sh was not found from the CLI build, the working directory, or RULEMETRIC_HARBOR_DIR. This job should not have been enqueued here: the readiness probe reports harness=false for a worker like this one.");let g=d(e,r.task),m=F(e,r.instructions);if(!n.exists(g))throw new Error(`harbor_ab: task directory not found: ${g}`);if(!n.exists(m))throw new Error(`harbor_ab: instruction file not found: ${m}`);let _=n.now(),s=d(n.jobsRoot,`${r.task}-${_}`);n.mkdir(s);let w=tr(n,s);l(`[harbor_ab] ${r.task} (${r.class}/${r.axis}, lane ${r.lane}): n=${r.nTrials}/arm on ${r.model} \u2192 ${s}`);let $=[d(e,"run.sh"),"ab",g,"--instructions",m,"--engine",a,"-m",r.model,"--n-attempts",String(r.nTrials)],p=await n.run("bash",$,{cwd:e,env:w,timeoutMs:Q,onLine:t(u=>l(`[harbor_ab] ${u}`),"onLine")});if(p.status!==0)throw new Error(`harbor_ab: ${r.task} run.sh exited ${p.status??"by signal"} after ${Math.round((n.now()-_)/1e3)}s: ${f(p.stderr)||f(p.stdout)||"(no output)"}`);let A=n.readdir(s),k=A.find(u=>/^ab-\d+-baseline$/.test(u)),E=A.find(u=>/^ab-\d+-with$/.test(u));if(!k||!E)throw new Error(`harbor_ab: run.sh exited 0 but the arm job dirs are missing under ${s} (found: ${A.join(", ")||"nothing"})`);let x=d(s,k),S=d(s,E),D=t(async u=>{let c=await j(n,e,w,"summarize.py",[u]);if(!c.json)throw new Error(`harbor_ab: summarize.py failed on ${u}: ${f(c.stderr,5)}`);return c.json},"summarize"),[N,U]=await Promise.all([D(x),D(S)]),R=await j(n,e,w,"cost_delta.py",[x,S,"--json"]);if(R.status!==0&&R.status!==3)throw new Error(`harbor_ab: cost_delta.py exited ${R.status}: ${f(R.stderr,5)}`);let b=R.json,h=await j(n,e,w,"provenance.py",["--compare",x,S,"--json"],[1]);if(h.status!==0&&h.status!==1)throw new Error(`harbor_ab: provenance.py exited ${h.status}: ${f(h.stderr,5)}`);if(!h.json)throw new Error(`harbor_ab: provenance.py printed no comparison: ${f(h.stderr,5)}`);let M=h.json,B=t((u,c)=>{let C=er(c);return{job:u,rewardMean:c.reward_mean,n:C.n,passes:C.passes,costUsd:c.cost_usd,tokens:c.tokens??null,durationMs:c.duration_ms,turns:c.turns,agentErrors:c.agent_errors}},"arm"),H=B(k,N),P=B(E,U),v=I({taskClass:r.class,axis:r.axis,baseline:{n:H.n,passes:H.passes},with:{n:P.n,passes:P.passes},cost:b,provenance:M});return l(`[harbor_ab] ${r.task}: ${v.decision} \u2014 ${v.reason}`),{task:r.task,class:r.class,axis:r.axis,lane:r.lane,model:r.model,...i?{measurement:i}:{},nTrials:r.nTrials,decision:v.decision,reason:v.reason,baseline:H,with:P,rewardDelta:v.reward.delta,rewardCi:v.reward.ci,cost:b?{decision:b.decision,deltaUsd:b.delta_usd,deltaPct:b.delta_pct,ciUsd:b.ci_usd,perSuccess:b.per_success?{decision:b.per_success.decision,deltaPct:b.per_success.delta_pct}:null,rewardRegressed:b.reward.regressed}:null,provenance:M,jobsDir:s,durationMs:n.now()-_}}t(wr,"harborAb");export{Q as a,er as b,wr as c};

package/dist/chunk-UJ7EPLKC.js

@@ -1 +0,0 @@
-import{d as _}from"./chunk-PUX6P5KC.js";import{d as D}from"./chunk-SG4UDDHB.js";import{c as P,d as w}from"./chunk-SBGXIPRN.js";import{a as l}from"./chunk-ZLLER3HP.js";import{spawn as L}from"node:child_process";var E=["claude","codex","pi"];function A(e,t=process.env){if(e){if(E.includes(e))return e;throw new Error(`Eval engine "${String(e)}" is not known (expected one of: ${E.join(", ")})`)}let o=t.RULEMETRIC_EVAL_ENGINE?.trim().toLowerCase();if(!o)return"claude";if(E.includes(o))return o;throw new Error(`RULEMETRIC_EVAL_ENGINE="${t.RULEMETRIC_EVAL_ENGINE}" is not a known engine (expected one of: ${E.join(", ")})`)}l(A,"resolveEvalEngine");function N(e,t=process.env){return e?.trim()||t.RULEMETRIC_CODEX_MODEL?.trim()||w().codexConfiguredModel||P}l(N,"resolveCodexEvalModel");function h(e){try{let t=JSON.parse(e),o=t.result??t.content??e,u=t.usage?.total_tokens??(t.usage?.input_tokens??0)+(t.usage?.output_tokens??0),r;if(t.modelUsage&&typeof t.modelUsage=="object"){let n=-1;for(let[a,d]of Object.entries(t.modelUsage)){let s=typeof d?.outputTokens=="number"?d.outputTokens:0;s>n&&(n=s,r=a)}}return{output:o,totalTokens:u,model:r}}catch{return{output:e,totalTokens:0}}}l(h,"parseClaudeJsonOutput");function S(e){let{prompt:t,targetContent:o,systemPrompt:u,model:r}=e,n=["-p",t,"--output-format","json","--settings",JSON.stringify({disableAllHooks:!0})],a=o??u;return a&&n.push("--system-prompt",a),r&&n.push("--model",r),n}l(S,"buildClaudeArgs");function U(e,t,o,u){let r=t.trim()||o.trim()||u.trim()||"(no output on stderr or stdout)";return`Exit code ${e}: ${r.slice(0,1e3)}`}l(U,"describeExitFailure");async function I(e){let{prompt:t,targetContent:o,systemPrompt:u,workDir:r,timeout:n,model:a}=e,d=o??u,s=await D({prompt:t,systemPrompt:d,model:N(a),timeout:n,workDir:r});return{output:s.output,exitCode:s.exitCode,durationMs:s.durationMs,totalTokens:s.usage?.totalTokens??0,model:s.model,engine:"codex",delivery:d?"prompt_prefix":"none",error:s.error}}l(I,"executeCodexArm");async function $(e){let t=e.targetContent??e.systemPrompt,o=await _({prompt:e.prompt,systemPrompt:t,model:e.model,timeout:e.timeout,workDir:e.workDir});return{output:o.output,exitCode:o.exitCode,durationMs:o.durationMs,totalTokens:o.usage?.totalTokens??0,model:o.model,provider:o.provider,costUsd:o.costUsd,engine:"pi",delivery:t?"system_prompt":"none",error:o.error}}l($,"executePiArm");async function H(e){let{prompt:t,targetContent:o,systemPrompt:u,workDir:r=process.cwd(),timeout:n=12e4,model:a}=e,d=A(e.engine);if(d==="codex")return I({...e,workDir:r,timeout:n});if(d==="pi")return $({...e,workDir:r,timeout:n});let s=S({prompt:t,targetContent:o,systemPrompt:u,model:a}),g=o??u?"system_prompt":"none",f=Date.now();return new Promise(v=>{let p={...process.env};delete p.CLAUDECODE,delete p.HTTPS_PROXY;let b=["/opt/homebrew/bin","/usr/local/bin"],x=(p.PATH??"").split(":").filter(Boolean),y=b.filter(i=>!x.includes(i));y.length&&(p.PATH=[...x,...y].join(":"));let m=L("claude",s,{cwd:r,env:p,stdio:["ignore","pipe","pipe"]}),c="",C="";m.stdout.on("data",i=>{c+=i.toString()}),m.stderr.on("data",i=>{C+=i.toString()});let T=setTimeout(()=>{m.kill("SIGTERM"),v({output:c,exitCode:null,durationMs:Date.now()-f,totalTokens:0,engine:"claude",delivery:g,error:`Timed out after ${n}ms`})},n);m.on("close",i=>{clearTimeout(T);let M=Date.now()-f,{output:k,totalTokens:O,model:R}=h(c);v({output:k,exitCode:i,durationMs:M,totalTokens:O,model:R,engine:"claude",delivery:g,error:i!==0?U(i,C,k,c):void 0})}),m.on("error",i=>{clearTimeout(T),v({output:"",exitCode:null,durationMs:Date.now()-f,totalTokens:0,engine:"claude",delivery:g,error:`Failed to spawn claude: ${i.message}`})})})}l(H,"executeEval");export{A as a,N as b,h as c,S as d,U as e,H as f};

package/dist/chunk-WSWZPNLG.js

@@ -1 +0,0 @@
-import{a as k}from"./chunk-OO2ZRKZY.js";import{b as P}from"./chunk-VBXL7XO7.js";import{f as b}from"./chunk-UJ7EPLKC.js";import{b as g,e as f,h}from"./chunk-ZZHIBOMT.js";import{a as c}from"./chunk-ZLLER3HP.js";var v=c(u=>(u.summary.pass_rate??0)>=1?1:0,"fullPassScore");async function _(u,a={}){let y=a.executeEval??b,w=a.grade??k,T=a.getExperiment??(async r=>{try{return await f(`/api/memory-experiments/${r}`)}catch(t){if(t instanceof g&&t.statusCode===404)return null;throw t}}),M=a.getMemoryBody??(async r=>{try{return(await f(`/api/memories/${r}`)).body}catch(t){if(t instanceof g&&t.statusCode===404)return null;throw t}}),l=a.updateExperiment??(async(r,t)=>{await h(`/api/memory-experiments/${r}`,t)}),e=await T(u.experimentId);if(!e)return{skipped:"not_found"};if(e.status==="completed"||e.status==="failed")return{skipped:"already_terminal",status:e.status};let x=await M(e.memoryId);if(x==null)return await l(e.id,{status:"failed",error:"memory no longer exists"}),{skipped:"memory_gone",status:"failed"};await l(e.id,{status:"running"});let E=e.expectations??[],d=e.model??void 0,i=e.engine??"claude",s=[],m=new Set,p=0;try{for(let t=0;t<e.trials;t++){let o=await y({prompt:e.taskPrompt,targetContent:x,model:d,engine:i}),n=await y({prompt:e.taskPrompt,model:d,engine:i});if(o.error||n.error){p++;continue}if(o.engine&&o.engine!==i||n.engine&&n.engine!==i||o.model&&n.model&&o.model!==n.model){p++;continue}o.model&&m.add(o.model),n.model&&m.add(n.model);let[C,U]=await Promise.all([w({prompt:e.taskPrompt,output:o.output,expectations:E,model:d}),w({prompt:e.taskPrompt,output:n.output,expectations:E,model:d})]);s.push({withScore:v(C),withoutScore:v(U),withLatency:o.durationMs/1e3,withoutLatency:n.durationMs/1e3,withTokens:o.totalTokens,withoutTokens:n.totalTokens,...o.costUsd!==void 0?{withCostUsd:o.costUsd}:{},...n.costUsd!==void 0?{withoutCostUsd:n.costUsd}:{}})}if(s.length===0){let t=`no trials completed (${p}/${e.trials} runs errored \u2014 is the ${i} CLI available?)`;return await l(e.id,{status:"failed",error:t}),{status:"failed",requestedTrials:e.trials,completedTrials:0}}let r=P(s);return await l(e.id,{status:"completed",result:{verdict:r,pairs:s,completedTrials:s.length,engine:i,model:m.size===1?[...m][0]:d??null}}),{status:"completed",requestedTrials:e.trials,completedTrials:s.length,significant:r.significant,delta:r.delta}}catch(r){let t=r instanceof Error?r.message:String(r);return await l(e.id,{status:"failed",error:t.slice(0,500)}).catch(()=>{}),{status:"failed",requestedTrials:e.trials,completedTrials:s.length}}}c(_,"processMemoryAb");export{_ as a};