@rulemetric/cli 0.7.24 → 0.7.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BSXUH5FD.js +1 -0
- package/dist/{chunk-Q6KLSCDY.js → chunk-KG3TW5QB.js} +1 -1
- package/dist/{chunk-RWJDEYOS.js → chunk-QYKLIRMY.js} +1 -1
- package/dist/commands/evals/agent.js +1 -1
- package/dist/commands/evals/validate-judge.js +1 -1
- package/dist/lib/agent-loop.js +1 -1
- package/dist/lib/handlers/process-judge-review.js +1 -1
- package/dist/lib/judge-review-runner.js +1 -1
- package/oclif.manifest.json +1 -1
- package/package.json +8 -8
- package/dist/chunk-LLBGUOG3.js +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-YDRZDBHG.js";import{c as J}from"./chunk-K25S2MGU.js";import{b as B,c as R,d as N,e as z}from"./chunk-3BVHPQZW.js";import{e as f,g as T}from"./chunk-H2ORWIEY.js";import{a as p}from"./chunk-ZLLER3HP.js";function W(r,a,s){return s?r.reviewerModel===s||r.reviewerModel===`${a}:${s}`:r.reviewerEngine===a}p(W,"reviewMatchesReviewer");var C=100;function Z(r){let a=new Set;return r.forEach((s,d)=>{s.method!=="deterministic"&&a.add(d)}),a}p(Z,"llmExpectationIndexes");var w=class extends Error{constructor(s,d){super(d);this.reason=s}static{p(this,"JudgeReviewSkip")}};async function oe(r){let a=r.log??(()=>{}),s=r.warn??a,d=r.targetId,K=r.parallel??3,U=r.timeoutMs??12e4,v=await f(`/api/eval-targets/${d}`),u=(await f(`/api/eval-targets/${d}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new w("no-labels",`No human labels on "${v.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:r.engineFlag??"auto",modelFlag:r.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new w("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(r.skipReviewed){let e=await f(`/api/eval-targets/${d}/judge-reviews`),t=new Set(e.filter(i=>W(i,n.engine,r.modelFlag)).map(i=>`${i.evalRunId}:${i.expectationIndex}`));if(b=u.filter(i=>!t.has(`${i.evalRunId}:${i.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await f(`/api/eval-targets/${d}/runs?limit=${C}&offset=${e}`);for(let i of t)y.set(i.id,i);if(t.length<C)break}let q=await f(`/api/eval-targets/${d}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let x=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let i=Z(t),g=P.get(e)??new Set;[...i].every(h=>g.has(h))||(o.delete(e),x+=1)}if(x>0&&a(`${x} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:x};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&B(e.error)){if(n.model===R)throw new w("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);s(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new w("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${v.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let $=[],I=0,k=new Set,m=r.limit!=null?[...o.entries()].slice(0,Math.max(1,r.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=p(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,i]=m[e],g=y.get(t),h=g?.grading?.expectations,O=g?.outputs?.transcript,F=g?X.get(g.evalId):void 0;if(!g||!h?.length||!O||!F){I+=i.length,s(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:F,output:O,expectations:h.map(l=>l.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let l of i){let L=h[l.expectationIndex],A=M.expectations[l.expectationIndex];if(!L||!A||L.text!==l.expectationText){I+=1;continue}if(M.judge_model==null&&A.method==="llm"){I+=1;continue}$.push({runId:t,expectationIndex:l.expectationIndex,text:l.expectationText,humanPassed:D(l.verdict,l.judgePassed),judgeAPassed:l.judgePassed,judgeBPassed:A.passed})}a(` ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j($,I),_=[...k][0]??n.model??`${n.engine} (CLI default)`,E=0;if($.length>0)try{E=(await T(`/api/eval-targets/${d}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:$.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${E} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){s(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:v.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:E,alreadyReviewed:c,deferredPartialRuns:x}}p(oe,"runJudgeReviewPass");export{W as a,Z as b,w as c,oe as d};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{c as d,d as t}from"./chunk-BSXUH5FD.js";import{a as r}from"./chunk-ZLLER3HP.js";async function g(o,i){let{evalTargetId:s}=o;try{let e=await t({targetId:s,engineFlag:"auto",skipReviewed:!0,log:r(a=>i.logger.info(`[judge-review] ${a}`),"log"),warn:r(a=>i.logger.warn(`[judge-review] ${a}`),"warn")});return{evalTargetId:s,reviewerModel:e.reviewerModel,comparedPairs:e.report.pairs,disagreements:e.report.disagreements.length,persisted:e.persisted,alreadyReviewed:e.alreadyReviewed,deferredPartialRuns:e.deferredPartialRuns}}catch(e){if(e instanceof d)return i.logger.info(`[judge-review] skipped (${e.reason}): ${e.message}`),{evalTargetId:s,skipped:e.reason};throw e}}r(g,"processJudgeReview");export{g as a};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{c as N}from"./chunk-TIOKY2IA.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as S}from"./chunk-MYPDODRJ.js";import{a as h}from"./chunk-O23Z6LTP.js";import{a as I}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-
|
|
1
|
+
import{c as N}from"./chunk-TIOKY2IA.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as S}from"./chunk-MYPDODRJ.js";import{a as h}from"./chunk-O23Z6LTP.js";import{a as I}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-KG3TW5QB.js";import{b as E}from"./chunk-W2MOTOQE.js";import{a as M}from"./chunk-WNAYTVCO.js";import{a as H}from"./chunk-ARZJNTMV.js";import{a as A}from"./chunk-RSKQRVPJ.js";import{a as x}from"./chunk-DO3KHK2X.js";import{a as k}from"./chunk-UF6US7VU.js";import{a as v}from"./chunk-MSTF7A7D.js";import{b as R}from"./chunk-4HZWSFSY.js";import{a as T}from"./chunk-LVOLDUFC.js";import{b}from"./chunk-5RBLIVBF.js";import{d as w}from"./chunk-3BVHPQZW.js";import{b as f,e as _,f as p}from"./chunk-H2ORWIEY.js";import{a as s}from"./chunk-ZLLER3HP.js";var B={process_eval:h,process_insights:I,process_launch:E,process_send_message:A,process_conversation:S,process_session_goal:y,distill_memories:R,memory_ab:M,process_announcement:T,process_run_cleanup:H,cron_suggest_instructions:v,cron_refresh_skills:k,cron_refresh_pricing:x,process_judge_review:L},P=Object.keys(B),W=new Set(["process_session_goal","process_announcement","process_run_cleanup","distill_memories","memory_ab"]);function $(e=process.env){return e.DATABASE_URL?P:P.filter(n=>!W.has(n))}s($,"availableCapabilities");var F=3e4,J=3e4,U=3e4,C=250,D=5e3,K=`thin-agent-${process.pid}`,g=null;function G(e,n,r,l){let c;try{let d=w();c={claude:d.claudeBin!=null,codex:d.codexBin!=null,codexModel:d.codexConfiguredModel}}catch{}let i=s(d=>{p("/api/workers/heartbeat",{workerType:"cli-worker",clientName:K,clientVersion:l,workerVersion:"0.1.0",currentJobId:null,inFlightCount:0,lastError:null,status:d,metadata:{capabilities:e,mode:"thin-agent",llmEngines:c}}).then(m=>{let t=m?.advisory;t&&t.message!==g?(g=t.message,a(r,{event:"worker_advisory",code:t.code,message:t.message})):t||(g=null)}).catch(m=>{a(r,{event:"presence_heartbeat_error",error:m instanceof Error?m.message:String(m)})})},"send");i("alive");let o=setInterval(()=>i("alive"),U);n.addEventListener("abort",()=>{clearInterval(o),i("shutting_down")},{once:!0})}s(G,"startPresenceHeartbeat");function a(e,n){try{e(JSON.stringify({ts:new Date().toISOString(),...n}))}catch{e(`[agent-loop] log_error event=${n.event}`)}}s(a,"structuredLog");function V(e,n){return n.aborted?Promise.resolve(!1):e<=0?Promise.resolve(!0):new Promise(r=>{let l=setTimeout(()=>{n.removeEventListener("abort",c),r(!0)},e),c=s(()=>{clearTimeout(l),n.removeEventListener("abort",c),r(!1)},"onAbort");n.addEventListener("abort",c,{once:!0})})}s(V,"abortableSleep");async function q(e,n){let r=`capabilities=${encodeURIComponent(e.join(","))}&waitMs=${n}`,l=await _(`/api/work/next?${r}`);return l==null||typeof l=="string"?null:l}s(q,"pollForWork");function Q(e,n){return{logger:{info:s(r=>n(r),"info"),warn:s(r=>n(r),"warn"),error:s(r=>n(r),"error")},job:{attempts:e}}}s(Q,"makeHelpers");async function X(e,n,r){let l=B[e.kind],c=Date.now();if(a(r,{event:"claim",jobId:e.id,kind:e.kind,attempts:e.attempts}),!l){let o=`unknown kind: ${e.kind}`;a(r,{event:"fail",jobId:e.id,kind:e.kind,terminal:!0,error:o}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:o,terminal:!0}).catch(()=>{});return}let i=setInterval(()=>{a(r,{event:"heartbeat",jobId:e.id}),p(`/api/work/${e.id}/heartbeat`,{leaseToken:e.leaseToken}).catch(o=>{a(r,{event:"heartbeat_error",jobId:e.id,error:o instanceof Error?o.message:String(o)})})},n);try{let o=Q(e.attempts,r),d=await l(e.payload,o)??{};clearInterval(i),a(r,{event:"complete",jobId:e.id,kind:e.kind,durationMs:Date.now()-c}),await p(`/api/work/${e.id}/complete`,{leaseToken:e.leaseToken,result:d})}catch(o){clearInterval(i);let d=b(o),m=o instanceof Error?o.message:String(o);a(r,{event:"fail",jobId:e.id,kind:e.kind,durationMs:Date.now()-c,terminal:d,error:m}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:m,terminal:d}).catch(t=>{a(r,{event:"fail_post_error",jobId:e.id,error:t instanceof Error?t.message:String(t)})})}}s(X,"runOne");async function fe(e){let n=e.log??(()=>{}),r=e.pollWaitMs??F,l=e.heartbeatIntervalMs??J,c=e.capabilities??$(),i=C;a(n,{event:"agent_start",capabilities:c,pollWaitMs:r,heartbeatIntervalMs:l}),G(c,e.signal,n,e.version),N({signal:e.signal,log:n}).catch(t=>{a(n,{event:"tt_heartbeat_fatal",error:t instanceof Error?t.message:String(t)})});let o=5*6e4,d=0,m=s(()=>{if(!e.readInstalledVersion||!e.version)return!1;let t=Date.now();if(t-d<o)return!1;d=t;let u=e.readInstalledVersion();return!!(u&&u!==e.version)},"versionDrifted");for(;!e.signal.aborted;){if(m()){a(n,{event:"agent_version_drift",bootVersion:e.version,installedVersion:e.readInstalledVersion?.()??null}),e.onVersionDrift?.();break}let t=null;try{t=await q(c,r)}catch(u){let O=u instanceof f?u.statusCode:void 0;if(a(n,{event:"poll_error",status:O,error:u instanceof Error?u.message:String(u)}),!await V(i,e.signal))break;i=Math.min(i*2,D);continue}if(!t){if(a(n,{event:"poll_204"}),!await V(i,e.signal))break;i=Math.min(i*2,D);continue}i=C,await X(t,l,n)}a(n,{event:"agent_stop"})}s(fe,"runAgentLoop");export{B as a,P as b,$ as c,q as d,fe as e};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{e as N}from"../../chunk-
|
|
1
|
+
import{e as N}from"../../chunk-QYKLIRMY.js";import"../../chunk-TIOKY2IA.js";import"../../chunk-35ST2BX7.js";import"../../chunk-MYPDODRJ.js";import"../../chunk-O23Z6LTP.js";import"../../chunk-PRDYXO6Z.js";import"../../chunk-QFVGUZDO.js";import"../../chunk-KG3TW5QB.js";import"../../chunk-W2MOTOQE.js";import"../../chunk-WNAYTVCO.js";import"../../chunk-ARZJNTMV.js";import"../../chunk-6QKI756T.js";import"../../chunk-RSKQRVPJ.js";import"../../chunk-DO3KHK2X.js";import"../../chunk-UF6US7VU.js";import"../../chunk-MSTF7A7D.js";import"../../chunk-4HZWSFSY.js";import"../../chunk-LVOLDUFC.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-N7TGRFVZ.js";import{k as f}from"../../chunk-KZ65LEUH.js";import"../../chunk-GTOEWPIM.js";import"../../chunk-6V4WM73L.js";import"../../chunk-JUF4JLZQ.js";import"../../chunk-TDD7IOV2.js";import"../../chunk-7DFNBRLY.js";import"../../chunk-MODXEGCU.js";import"../../chunk-GQRCSR2A.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-CHA5SX4Y.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as _}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import{e as R,l as w}from"../../chunk-O6C2UGBY.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as z,context as Q}from"@opentelemetry/api";import{NodeSDK as A}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as I}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as L,ConsoleSpanExporter as C,SimpleSpanProcessor as D}from"@opentelemetry/sdk-trace-node";import{Resource as M}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as V,ATTR_SERVICE_VERSION as q}from"@opentelemetry/semantic-conventions";function b(t){if(!t)return{};let r={};for(let n of t.split(",")){let o=n.indexOf("=");if(o===-1)continue;let s=n.slice(0,o).trim(),i=n.slice(o+1).trim();s&&(r[s]=i)}return r}e(b,"parseOtlpHeaders");var c=null;function g(){return process.env.TELEMETRY_ENABLED!=="false"}e(g,"isTelemetryEnabled");function E(t){if(!g()||c!==null)return"off";let r=new M({[V]:t.serviceName,[q]:t.serviceVersion??"0.1.0"}),n=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,o,s;if(n){let i=b(process.env.OTEL_EXPORTER_OTLP_HEADERS),u=new I({url:`${n}/v1/traces`,headers:i});s=new L(u),o="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")s=void 0,o="silent";else{let i=new C;s=new D(i),o="console"}return c=new A({resource:r,spanProcessors:s?[s]:[]}),c.start(),o}e(E,"initTelemetry");async function d(){c&&(await c.shutdown(),c=null)}e(d,"shutdownTelemetry");import{SpanKind as ae,SpanStatusCode as ce}from"@opentelemetry/api";import{trace as de,context as me}from"@opentelemetry/api";import*as l from"@sentry/node";var h=!1;function y(t){if(h)return!1;let r=process.env.SENTRY_DSN;return r?(l.init({dsn:r,serverName:t.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),h=!0,!0):!1}e(y,"initSentry");async function m(t){h&&await l.flush(t)}e(m,"flushSentry");import{existsSync as G,mkdirSync as U,readFileSync as v,unlinkSync as k,writeFileSync as O}from"node:fs";import{dirname as X,join as P}from"node:path";import{execFileSync as $}from"node:child_process";var a=P(w(),"evals-agent.lock");function j(t){try{return process.kill(t,0),!0}catch(r){return r.code==="EPERM"}}e(j,"pidAlive");function B(t){if(!j(t))return!1;try{let r=$("ps",["-p",String(t),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(r)}catch{return!0}}e(B,"isLiveAgent");function H(){try{U(X(a),{recursive:!0})}catch{}for(let t=0;t<2;t++)try{return O(a,String(process.pid),{flag:"wx"}),{ok:!0}}catch(r){if(r.code!=="EEXIST")throw r;let n=0;try{n=Number(v(a,"utf-8").trim())}catch{}if(n&&n!==process.pid&&B(n))return{ok:!1,holder:n};try{k(a)}catch{}}try{O(a,String(process.pid))}catch{}return{ok:!0}}e(H,"acquireAgentLock");function S(){try{G(a)&&Number(v(a,"utf-8").trim())===process.pid&&k(a)}catch{}}e(S,"releaseAgentLock");var T=class t extends _{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(t),this.requireAuth();let r=H();if(!r.ok){this.log(`Another rulemetric evals agent is already running (pid ${r.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",y({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let o=E({serviceName:"rulemetric-agent"});o==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):o==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!R()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
|
|
2
2
|
`);let i=new AbortController,u=e(async p=>{this.log(`
|
|
3
3
|
[${p}] shutting down agent...`),i.abort(),await Promise.allSettled([m(2e3),d(),f()]),S(),process.exit(0)},"shutdown");process.on("SIGINT",()=>{u("SIGINT")}),process.on("SIGTERM",()=>{u("SIGTERM")});let x=!1;try{await N({signal:i.signal,log:e(p=>this.log(p),"log"),version:this.config.version,readInstalledVersion:e(()=>{try{return JSON.parse(v(P(this.config.root,"package.json"),"utf-8")).version??null}catch{return null}},"readInstalledVersion"),onVersionDrift:e(()=>{x=!0},"onVersionDrift")}),x&&(this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code."),await Promise.allSettled([m(2e3),d(),f()]),S(),process.exit(1))}finally{S(),await f()}}};export{T as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{c,d as m}from"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as u}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as r}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as s}from"@oclif/core";var o=r(n=>n==null?"\u2014":`${(n*100).toFixed(0)}%`,"pct"),g=class n extends u{static{r(this,"EvalValidateJudge")}static description="Score a second model against your human judge-review labels (\xA76g validation pass)";static examples=["<%= config.bin %> evals validate-judge <target-id> # auto-detect installed CLIs","<%= config.bin %> evals validate-judge <target-id> --engine codex --model gpt-5.5","<%= config.bin %> evals validate-judge <target-id> --model claude-sonnet-5 --json","<%= config.bin %> evals validate-judge <target-id> --limit 5 # quick sample"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={model:s.string({description:"Second reviewer model. Default: resolved from your installed tooling (codex: your ~/.codex/config.toml model; claude: a tier that differs from the original judge)."}),engine:s.string({description:"CLI that runs the second reviewer. `auto` prefers your installed codex for a cross-vendor reviewer \u2014 two Claude models share a training lineage, so their agreement overstates independence.",options:["auto","claude","codex"],default:"auto"}),limit:s.integer({description:"Sample only the N most-recently-annotated runs (cost control / smoke)"}),"skip-reviewed":s.boolean({description:"Only re-grade assertions this reviewer model has not ruled on yet",default:!1}),parallel:s.integer({description:"Concurrent re-grades",default:3}),timeout:s.integer({description:"Timeout per re-grade in ms",default:12e4}),json:s.boolean({description:"Emit the full report as JSON"})};async run(){let{args:h,flags:i}=await this.parse(n);this.requireAuth();let a;try{a=await m({targetId:h.targetId,engineFlag:i.engine,modelFlag:i.model,limit:i.limit,parallel:i.parallel,timeoutMs:i.timeout,skipReviewed:i["skip-reviewed"],log:r(t=>this.log(t),"log"),warn:r(t=>this.warn(t),"warn")})}catch(t){throw t instanceof c&&this.error(t.message),t}let{report:e,judgeModels:l,reviewerModel:d}=a;if(l.length===1&&l[0]===d&&this.warn(`Second reviewer resolved to the SAME model as the original judge (${l[0]}) \u2014 this measures self-consistency, not independent review.`),i.json){this.log(JSON.stringify({target:a.targetName,judgeA:l,judgeB:d,persisted:a.persisted,alreadyReviewed:a.alreadyReviewed,...e},null,2));return}if(e.pairs===0&&a.alreadyReviewed>0){this.log(`All ${a.alreadyReviewed} labeled assertion(s) already reviewed by ${d} \u2014 nothing new to grade.`);return}if(this.log(""),this.log(`Second reviewer: ${d} \u2014 scored against ${e.pairs} human-labeled assertion(s)`+(e.skipped?` (${e.skipped} skipped)`:"")),this.log(""),this.log(` reviewer vs HUMAN: ${o(e.bVsHuman.overall.rate)} overall \xB7 ${o(e.bVsHuman.onJudgePass.rate)} where judge passed \xB7 ${o(e.bVsHuman.onJudgeFail.rate)} where judge failed`),this.log(` original judge vs human: ${o(e.aVsHuman.overall.rate)} (your approve rate)`),this.log(` reviewer vs judge: ${o(e.bVsA.overall.rate)} (inter-model, not truth)`),this.log(` Cohen's \u03BA (reviewer vs human): ${e.kappaBvsHuman==null?"undefined (degenerate marginals)":e.kappaBvsHuman.toFixed(2)}`),this.log(""),e.suggestedAnchorWeight!=null&&this.log(` suggested \xA76g anchor weight if used as a passive label source: ${e.suggestedAnchorWeight.toFixed(2)} (misclassification attenuation (2\xB7acc\u22121)\xB2; validated:false either way)`),e.disagreements.length>0){this.log(""),this.log(` ${e.disagreements.length} disagreement(s) with your labels \u2014 flagged on the Review tab:`);for(let t of e.disagreements.slice(0,10))this.log(` run ${t.runId.slice(0,8)} #${t.expectationIndex}: human=${t.humanPassed?"pass":"fail"} reviewer=${t.judgeBPassed?"pass":"fail"} \u2014 ${t.text.slice(0,80)}`);e.disagreements.length>10&&this.log(` \u2026 and ${e.disagreements.length-10} more (use --json for all)`)}}};export{g as default};
|
package/dist/lib/agent-loop.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c,d,e}from"../chunk-
|
|
1
|
+
import{a,b,c,d,e}from"../chunk-QYKLIRMY.js";import"../chunk-TIOKY2IA.js";import"../chunk-35ST2BX7.js";import"../chunk-MYPDODRJ.js";import"../chunk-O23Z6LTP.js";import"../chunk-PRDYXO6Z.js";import"../chunk-QFVGUZDO.js";import"../chunk-KG3TW5QB.js";import"../chunk-W2MOTOQE.js";import"../chunk-WNAYTVCO.js";import"../chunk-ARZJNTMV.js";import"../chunk-6QKI756T.js";import"../chunk-RSKQRVPJ.js";import"../chunk-DO3KHK2X.js";import"../chunk-UF6US7VU.js";import"../chunk-MSTF7A7D.js";import"../chunk-4HZWSFSY.js";import"../chunk-LVOLDUFC.js";import"../chunk-5RBLIVBF.js";import"../chunk-ZL4MZMUK.js";import"../chunk-N7TGRFVZ.js";import"../chunk-KZ65LEUH.js";import"../chunk-GTOEWPIM.js";import"../chunk-6V4WM73L.js";import"../chunk-JUF4JLZQ.js";import"../chunk-TDD7IOV2.js";import"../chunk-7DFNBRLY.js";import"../chunk-MODXEGCU.js";import"../chunk-GQRCSR2A.js";import"../chunk-UUMW6X2W.js";import"../chunk-BSXUH5FD.js";import"../chunk-6UEFCMR2.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-CHA5SX4Y.js";import"../chunk-WQ2BJQYB.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-KG3TW5QB.js";import"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processJudgeReview};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c}from"../chunk-
|
|
1
|
+
import{a,b,c,d}from"../chunk-BSXUH5FD.js";import"../chunk-6UEFCMR2.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-WQ2BJQYB.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{c as JudgeReviewSkip,b as llmExpectationIndexes,a as reviewMatchesReviewer,d as runJudgeReviewPass};
|
package/oclif.manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulemetric/cli",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.25",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -74,9 +74,9 @@
|
|
|
74
74
|
"postgres": "^3.4.0",
|
|
75
75
|
"react": "^19.0.0",
|
|
76
76
|
"zod": "^3.24.0",
|
|
77
|
-
"@rulemetric/hooks": "0.7.
|
|
78
|
-
"@rulemetric/
|
|
79
|
-
"@rulemetric/
|
|
77
|
+
"@rulemetric/hooks": "0.7.25",
|
|
78
|
+
"@rulemetric/proxy": "0.7.25",
|
|
79
|
+
"@rulemetric/skills-registry": "0.7.25"
|
|
80
80
|
},
|
|
81
81
|
"//deps": "Worker (`evals agent`) runtime deps: the @opentelemetry/* set, @sentry/node, and postgres are STATIC top-level imports loaded at worker startup (via bundled @rulemetric/{telemetry,db}) — they MUST be runtime deps or a clean `npm i -g` crashes with ModuleLoadError. `better-sqlite3` is optionalDependencies (below): it's a native module, lazy `require()`'d inside a try/catch (packages/session providers cursor/opencode) so a missing binary degrades gracefully — making it a hard dep would break `npm i -g` on platforms without prebuilt binaries. `resend` stays a devDep: lazy `await import('resend')` guarded by RESEND_API_KEY, which end-user workers never set (provider falls back to noop). Classification verified via import-graph scan of dist/commands/evals/agent.js + per-importer static/lazy check.",
|
|
82
82
|
"optionalDependencies": {
|
|
@@ -94,14 +94,14 @@
|
|
|
94
94
|
"tsx": "^4.19.0",
|
|
95
95
|
"typescript": "^5.9.0",
|
|
96
96
|
"vitest": "^3.2.0",
|
|
97
|
-
"@rulemetric/core": "0.0.1",
|
|
98
97
|
"@rulemetric/api": "0.0.1",
|
|
99
|
-
"@rulemetric/db": "0.0.1",
|
|
100
98
|
"@rulemetric/effectiveness": "0.0.1",
|
|
101
99
|
"@rulemetric/email": "0.0.1",
|
|
102
100
|
"@rulemetric/session": "0.2.1",
|
|
103
|
-
"@rulemetric/
|
|
104
|
-
"@rulemetric/
|
|
101
|
+
"@rulemetric/core": "0.0.1",
|
|
102
|
+
"@rulemetric/db": "0.0.1",
|
|
103
|
+
"@rulemetric/typescript-config": "0.0.0",
|
|
104
|
+
"@rulemetric/telemetry": "0.0.1"
|
|
105
105
|
},
|
|
106
106
|
"scripts": {
|
|
107
107
|
"build": "node scripts/build.mjs",
|
package/dist/chunk-LLBGUOG3.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-YDRZDBHG.js";import{c as J}from"./chunk-K25S2MGU.js";import{b as L,c as R,d as N,e as z}from"./chunk-3BVHPQZW.js";import{e as p,g as B}from"./chunk-H2ORWIEY.js";import{a as h}from"./chunk-ZLLER3HP.js";var C=100;function W(i){let a=new Set;return i.forEach((g,s)=>{g.method!=="deterministic"&&a.add(s)}),a}h(W,"llmExpectationIndexes");var f=class extends Error{constructor(g,s){super(s);this.reason=g}static{h(this,"JudgeReviewSkip")}};async function ae(i){let a=i.log??(()=>{}),g=i.warn??a,s=i.targetId,K=i.parallel??3,U=i.timeoutMs??12e4,w=await p(`/api/eval-targets/${s}`),u=(await p(`/api/eval-targets/${s}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new f("no-labels",`No human labels on "${w.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:i.engineFlag??"auto",modelFlag:i.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new f("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(i.skipReviewed&&n.model){let e=await p(`/api/eval-targets/${s}/judge-reviews`),t=new Set(e.filter(r=>r.reviewerModel===n.model).map(r=>`${r.evalRunId}:${r.expectationIndex}`));if(b=u.filter(r=>!t.has(`${r.evalRunId}:${r.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:w.name,reviewer:n,judgeModels:[],reviewerModel:n.model,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await p(`/api/eval-targets/${s}/runs?limit=${C}&offset=${e}`);for(let r of t)y.set(r.id,r);if(t.length<C)break}let q=await p(`/api/eval-targets/${s}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let v=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let r=W(t),l=P.get(e)??new Set;[...r].every(x=>l.has(x))||(o.delete(e),v+=1)}if(v>0&&a(`${v} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:w.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:v};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&L(e.error)){if(n.model===R)throw new f("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);g(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new f("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${w.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let I=[],$=0,k=new Set,m=i.limit!=null?[...o.entries()].slice(0,Math.max(1,i.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=h(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,r]=m[e],l=y.get(t),x=l?.grading?.expectations,F=l?.outputs?.transcript,O=l?X.get(l.evalId):void 0;if(!l||!x?.length||!F||!O){$+=r.length,g(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:O,output:F,expectations:x.map(d=>d.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let d of r){let T=x[d.expectationIndex],E=M.expectations[d.expectationIndex];if(!T||!E||T.text!==d.expectationText){$+=1;continue}if(M.judge_model==null&&E.method==="llm"){$+=1;continue}I.push({runId:t,expectationIndex:d.expectationIndex,text:d.expectationText,humanPassed:D(d.verdict,d.judgePassed),judgeAPassed:d.judgePassed,judgeBPassed:E.passed})}a(` ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j(I,$),_=[...k][0]??n.model??`${n.engine} (CLI default)`,A=0;if(I.length>0)try{A=(await B(`/api/eval-targets/${s}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:I.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${A} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){g(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:w.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:A,alreadyReviewed:c,deferredPartialRuns:v}}h(ae,"runJudgeReviewPass");export{W as a,f as b,ae as c};
|