@rulemetric/cli 0.7.24 → 0.7.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BSXUH5FD.js +1 -0
- package/dist/{chunk-Q6KLSCDY.js → chunk-KG3TW5QB.js} +1 -1
- package/dist/{chunk-RWJDEYOS.js → chunk-QYKLIRMY.js} +1 -1
- package/dist/commands/evals/agent.js +1 -1
- package/dist/commands/evals/validate-judge.js +1 -1
- package/dist/lib/agent-loop.js +1 -1
- package/dist/lib/handlers/process-judge-review.js +1 -1
- package/dist/lib/judge-review-runner.js +1 -1
- package/oclif.manifest.json +83 -83
- package/package.json +5 -5
- package/dist/chunk-LLBGUOG3.js +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-YDRZDBHG.js";import{c as J}from"./chunk-K25S2MGU.js";import{b as B,c as R,d as N,e as z}from"./chunk-3BVHPQZW.js";import{e as f,g as T}from"./chunk-H2ORWIEY.js";import{a as p}from"./chunk-ZLLER3HP.js";function W(r,a,s){return s?r.reviewerModel===s||r.reviewerModel===`${a}:${s}`:r.reviewerEngine===a}p(W,"reviewMatchesReviewer");var C=100;function Z(r){let a=new Set;return r.forEach((s,d)=>{s.method!=="deterministic"&&a.add(d)}),a}p(Z,"llmExpectationIndexes");var w=class extends Error{constructor(s,d){super(d);this.reason=s}static{p(this,"JudgeReviewSkip")}};async function oe(r){let a=r.log??(()=>{}),s=r.warn??a,d=r.targetId,K=r.parallel??3,U=r.timeoutMs??12e4,v=await f(`/api/eval-targets/${d}`),u=(await f(`/api/eval-targets/${d}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new w("no-labels",`No human labels on "${v.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:r.engineFlag??"auto",modelFlag:r.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new w("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(r.skipReviewed){let e=await f(`/api/eval-targets/${d}/judge-reviews`),t=new Set(e.filter(i=>W(i,n.engine,r.modelFlag)).map(i=>`${i.evalRunId}:${i.expectationIndex}`));if(b=u.filter(i=>!t.has(`${i.evalRunId}:${i.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await f(`/api/eval-targets/${d}/runs?limit=${C}&offset=${e}`);for(let i of t)y.set(i.id,i);if(t.length<C)break}let q=await f(`/api/eval-targets/${d}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let x=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let i=Z(t),g=P.get(e)??new Set;[...i].every(h=>g.has(h))||(o.delete(e),x+=1)}if(x>0&&a(`${x} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:v.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:x};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&B(e.error)){if(n.model===R)throw new w("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);s(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new w("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${v.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let $=[],I=0,k=new Set,m=r.limit!=null?[...o.entries()].slice(0,Math.max(1,r.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=p(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,i]=m[e],g=y.get(t),h=g?.grading?.expectations,O=g?.outputs?.transcript,F=g?X.get(g.evalId):void 0;if(!g||!h?.length||!O||!F){I+=i.length,s(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:F,output:O,expectations:h.map(l=>l.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let l of i){let L=h[l.expectationIndex],A=M.expectations[l.expectationIndex];if(!L||!A||L.text!==l.expectationText){I+=1;continue}if(M.judge_model==null&&A.method==="llm"){I+=1;continue}$.push({runId:t,expectationIndex:l.expectationIndex,text:l.expectationText,humanPassed:D(l.verdict,l.judgePassed),judgeAPassed:l.judgePassed,judgeBPassed:A.passed})}a(` ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j($,I),_=[...k][0]??n.model??`${n.engine} (CLI default)`,E=0;if($.length>0)try{E=(await T(`/api/eval-targets/${d}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:$.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${E} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){s(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:v.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:E,alreadyReviewed:c,deferredPartialRuns:x}}p(oe,"runJudgeReviewPass");export{W as a,Z as b,w as c,oe as d};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{c as d,d as t}from"./chunk-BSXUH5FD.js";import{a as r}from"./chunk-ZLLER3HP.js";async function g(o,i){let{evalTargetId:s}=o;try{let e=await t({targetId:s,engineFlag:"auto",skipReviewed:!0,log:r(a=>i.logger.info(`[judge-review] ${a}`),"log"),warn:r(a=>i.logger.warn(`[judge-review] ${a}`),"warn")});return{evalTargetId:s,reviewerModel:e.reviewerModel,comparedPairs:e.report.pairs,disagreements:e.report.disagreements.length,persisted:e.persisted,alreadyReviewed:e.alreadyReviewed,deferredPartialRuns:e.deferredPartialRuns}}catch(e){if(e instanceof d)return i.logger.info(`[judge-review] skipped (${e.reason}): ${e.message}`),{evalTargetId:s,skipped:e.reason};throw e}}r(g,"processJudgeReview");export{g as a};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{c as N}from"./chunk-TIOKY2IA.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as S}from"./chunk-MYPDODRJ.js";import{a as h}from"./chunk-O23Z6LTP.js";import{a as I}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-
|
|
1
|
+
import{c as N}from"./chunk-TIOKY2IA.js";import{a as y}from"./chunk-35ST2BX7.js";import{a as S}from"./chunk-MYPDODRJ.js";import{a as h}from"./chunk-O23Z6LTP.js";import{a as I}from"./chunk-PRDYXO6Z.js";import{a as L}from"./chunk-KG3TW5QB.js";import{b as E}from"./chunk-W2MOTOQE.js";import{a as M}from"./chunk-WNAYTVCO.js";import{a as H}from"./chunk-ARZJNTMV.js";import{a as A}from"./chunk-RSKQRVPJ.js";import{a as x}from"./chunk-DO3KHK2X.js";import{a as k}from"./chunk-UF6US7VU.js";import{a as v}from"./chunk-MSTF7A7D.js";import{b as R}from"./chunk-4HZWSFSY.js";import{a as T}from"./chunk-LVOLDUFC.js";import{b}from"./chunk-5RBLIVBF.js";import{d as w}from"./chunk-3BVHPQZW.js";import{b as f,e as _,f as p}from"./chunk-H2ORWIEY.js";import{a as s}from"./chunk-ZLLER3HP.js";var B={process_eval:h,process_insights:I,process_launch:E,process_send_message:A,process_conversation:S,process_session_goal:y,distill_memories:R,memory_ab:M,process_announcement:T,process_run_cleanup:H,cron_suggest_instructions:v,cron_refresh_skills:k,cron_refresh_pricing:x,process_judge_review:L},P=Object.keys(B),W=new Set(["process_session_goal","process_announcement","process_run_cleanup","distill_memories","memory_ab"]);function $(e=process.env){return e.DATABASE_URL?P:P.filter(n=>!W.has(n))}s($,"availableCapabilities");var F=3e4,J=3e4,U=3e4,C=250,D=5e3,K=`thin-agent-${process.pid}`,g=null;function G(e,n,r,l){let c;try{let d=w();c={claude:d.claudeBin!=null,codex:d.codexBin!=null,codexModel:d.codexConfiguredModel}}catch{}let i=s(d=>{p("/api/workers/heartbeat",{workerType:"cli-worker",clientName:K,clientVersion:l,workerVersion:"0.1.0",currentJobId:null,inFlightCount:0,lastError:null,status:d,metadata:{capabilities:e,mode:"thin-agent",llmEngines:c}}).then(m=>{let t=m?.advisory;t&&t.message!==g?(g=t.message,a(r,{event:"worker_advisory",code:t.code,message:t.message})):t||(g=null)}).catch(m=>{a(r,{event:"presence_heartbeat_error",error:m instanceof Error?m.message:String(m)})})},"send");i("alive");let o=setInterval(()=>i("alive"),U);n.addEventListener("abort",()=>{clearInterval(o),i("shutting_down")},{once:!0})}s(G,"startPresenceHeartbeat");function a(e,n){try{e(JSON.stringify({ts:new Date().toISOString(),...n}))}catch{e(`[agent-loop] log_error event=${n.event}`)}}s(a,"structuredLog");function V(e,n){return n.aborted?Promise.resolve(!1):e<=0?Promise.resolve(!0):new Promise(r=>{let l=setTimeout(()=>{n.removeEventListener("abort",c),r(!0)},e),c=s(()=>{clearTimeout(l),n.removeEventListener("abort",c),r(!1)},"onAbort");n.addEventListener("abort",c,{once:!0})})}s(V,"abortableSleep");async function q(e,n){let r=`capabilities=${encodeURIComponent(e.join(","))}&waitMs=${n}`,l=await _(`/api/work/next?${r}`);return l==null||typeof l=="string"?null:l}s(q,"pollForWork");function Q(e,n){return{logger:{info:s(r=>n(r),"info"),warn:s(r=>n(r),"warn"),error:s(r=>n(r),"error")},job:{attempts:e}}}s(Q,"makeHelpers");async function X(e,n,r){let l=B[e.kind],c=Date.now();if(a(r,{event:"claim",jobId:e.id,kind:e.kind,attempts:e.attempts}),!l){let o=`unknown kind: ${e.kind}`;a(r,{event:"fail",jobId:e.id,kind:e.kind,terminal:!0,error:o}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:o,terminal:!0}).catch(()=>{});return}let i=setInterval(()=>{a(r,{event:"heartbeat",jobId:e.id}),p(`/api/work/${e.id}/heartbeat`,{leaseToken:e.leaseToken}).catch(o=>{a(r,{event:"heartbeat_error",jobId:e.id,error:o instanceof Error?o.message:String(o)})})},n);try{let o=Q(e.attempts,r),d=await l(e.payload,o)??{};clearInterval(i),a(r,{event:"complete",jobId:e.id,kind:e.kind,durationMs:Date.now()-c}),await p(`/api/work/${e.id}/complete`,{leaseToken:e.leaseToken,result:d})}catch(o){clearInterval(i);let d=b(o),m=o instanceof Error?o.message:String(o);a(r,{event:"fail",jobId:e.id,kind:e.kind,durationMs:Date.now()-c,terminal:d,error:m}),await p(`/api/work/${e.id}/fail`,{leaseToken:e.leaseToken,error:m,terminal:d}).catch(t=>{a(r,{event:"fail_post_error",jobId:e.id,error:t instanceof Error?t.message:String(t)})})}}s(X,"runOne");async function fe(e){let n=e.log??(()=>{}),r=e.pollWaitMs??F,l=e.heartbeatIntervalMs??J,c=e.capabilities??$(),i=C;a(n,{event:"agent_start",capabilities:c,pollWaitMs:r,heartbeatIntervalMs:l}),G(c,e.signal,n,e.version),N({signal:e.signal,log:n}).catch(t=>{a(n,{event:"tt_heartbeat_fatal",error:t instanceof Error?t.message:String(t)})});let o=5*6e4,d=0,m=s(()=>{if(!e.readInstalledVersion||!e.version)return!1;let t=Date.now();if(t-d<o)return!1;d=t;let u=e.readInstalledVersion();return!!(u&&u!==e.version)},"versionDrifted");for(;!e.signal.aborted;){if(m()){a(n,{event:"agent_version_drift",bootVersion:e.version,installedVersion:e.readInstalledVersion?.()??null}),e.onVersionDrift?.();break}let t=null;try{t=await q(c,r)}catch(u){let O=u instanceof f?u.statusCode:void 0;if(a(n,{event:"poll_error",status:O,error:u instanceof Error?u.message:String(u)}),!await V(i,e.signal))break;i=Math.min(i*2,D);continue}if(!t){if(a(n,{event:"poll_204"}),!await V(i,e.signal))break;i=Math.min(i*2,D);continue}i=C,await X(t,l,n)}a(n,{event:"agent_stop"})}s(fe,"runAgentLoop");export{B as a,P as b,$ as c,q as d,fe as e};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{e as N}from"../../chunk-
|
|
1
|
+
import{e as N}from"../../chunk-QYKLIRMY.js";import"../../chunk-TIOKY2IA.js";import"../../chunk-35ST2BX7.js";import"../../chunk-MYPDODRJ.js";import"../../chunk-O23Z6LTP.js";import"../../chunk-PRDYXO6Z.js";import"../../chunk-QFVGUZDO.js";import"../../chunk-KG3TW5QB.js";import"../../chunk-W2MOTOQE.js";import"../../chunk-WNAYTVCO.js";import"../../chunk-ARZJNTMV.js";import"../../chunk-6QKI756T.js";import"../../chunk-RSKQRVPJ.js";import"../../chunk-DO3KHK2X.js";import"../../chunk-UF6US7VU.js";import"../../chunk-MSTF7A7D.js";import"../../chunk-4HZWSFSY.js";import"../../chunk-LVOLDUFC.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-N7TGRFVZ.js";import{k as f}from"../../chunk-KZ65LEUH.js";import"../../chunk-GTOEWPIM.js";import"../../chunk-6V4WM73L.js";import"../../chunk-JUF4JLZQ.js";import"../../chunk-TDD7IOV2.js";import"../../chunk-7DFNBRLY.js";import"../../chunk-MODXEGCU.js";import"../../chunk-GQRCSR2A.js";import"../../chunk-UUMW6X2W.js";import"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-CHA5SX4Y.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as _}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import{e as R,l as w}from"../../chunk-O6C2UGBY.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as z,context as Q}from"@opentelemetry/api";import{NodeSDK as A}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as I}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as L,ConsoleSpanExporter as C,SimpleSpanProcessor as D}from"@opentelemetry/sdk-trace-node";import{Resource as M}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as V,ATTR_SERVICE_VERSION as q}from"@opentelemetry/semantic-conventions";function b(t){if(!t)return{};let r={};for(let n of t.split(",")){let o=n.indexOf("=");if(o===-1)continue;let s=n.slice(0,o).trim(),i=n.slice(o+1).trim();s&&(r[s]=i)}return r}e(b,"parseOtlpHeaders");var c=null;function g(){return process.env.TELEMETRY_ENABLED!=="false"}e(g,"isTelemetryEnabled");function E(t){if(!g()||c!==null)return"off";let r=new M({[V]:t.serviceName,[q]:t.serviceVersion??"0.1.0"}),n=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,o,s;if(n){let i=b(process.env.OTEL_EXPORTER_OTLP_HEADERS),u=new I({url:`${n}/v1/traces`,headers:i});s=new L(u),o="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")s=void 0,o="silent";else{let i=new C;s=new D(i),o="console"}return c=new A({resource:r,spanProcessors:s?[s]:[]}),c.start(),o}e(E,"initTelemetry");async function d(){c&&(await c.shutdown(),c=null)}e(d,"shutdownTelemetry");import{SpanKind as ae,SpanStatusCode as ce}from"@opentelemetry/api";import{trace as de,context as me}from"@opentelemetry/api";import*as l from"@sentry/node";var h=!1;function y(t){if(h)return!1;let r=process.env.SENTRY_DSN;return r?(l.init({dsn:r,serverName:t.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),h=!0,!0):!1}e(y,"initSentry");async function m(t){h&&await l.flush(t)}e(m,"flushSentry");import{existsSync as G,mkdirSync as U,readFileSync as v,unlinkSync as k,writeFileSync as O}from"node:fs";import{dirname as X,join as P}from"node:path";import{execFileSync as $}from"node:child_process";var a=P(w(),"evals-agent.lock");function j(t){try{return process.kill(t,0),!0}catch(r){return r.code==="EPERM"}}e(j,"pidAlive");function B(t){if(!j(t))return!1;try{let r=$("ps",["-p",String(t),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(r)}catch{return!0}}e(B,"isLiveAgent");function H(){try{U(X(a),{recursive:!0})}catch{}for(let t=0;t<2;t++)try{return O(a,String(process.pid),{flag:"wx"}),{ok:!0}}catch(r){if(r.code!=="EEXIST")throw r;let n=0;try{n=Number(v(a,"utf-8").trim())}catch{}if(n&&n!==process.pid&&B(n))return{ok:!1,holder:n};try{k(a)}catch{}}try{O(a,String(process.pid))}catch{}return{ok:!0}}e(H,"acquireAgentLock");function S(){try{G(a)&&Number(v(a,"utf-8").trim())===process.pid&&k(a)}catch{}}e(S,"releaseAgentLock");var T=class t extends _{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(t),this.requireAuth();let r=H();if(!r.ok){this.log(`Another rulemetric evals agent is already running (pid ${r.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",y({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let o=E({serviceName:"rulemetric-agent"});o==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):o==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!R()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
|
|
2
2
|
`);let i=new AbortController,u=e(async p=>{this.log(`
|
|
3
3
|
[${p}] shutting down agent...`),i.abort(),await Promise.allSettled([m(2e3),d(),f()]),S(),process.exit(0)},"shutdown");process.on("SIGINT",()=>{u("SIGINT")}),process.on("SIGTERM",()=>{u("SIGTERM")});let x=!1;try{await N({signal:i.signal,log:e(p=>this.log(p),"log"),version:this.config.version,readInstalledVersion:e(()=>{try{return JSON.parse(v(P(this.config.root,"package.json"),"utf-8")).version??null}catch{return null}},"readInstalledVersion"),onVersionDrift:e(()=>{x=!0},"onVersionDrift")}),x&&(this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code."),await Promise.allSettled([m(2e3),d(),f()]),S(),process.exit(1))}finally{S(),await f()}}};export{T as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{c,d as m}from"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import{a as u}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as r}from"../../chunk-ZLLER3HP.js";import{Args as p,Flags as s}from"@oclif/core";var o=r(n=>n==null?"\u2014":`${(n*100).toFixed(0)}%`,"pct"),g=class n extends u{static{r(this,"EvalValidateJudge")}static description="Score a second model against your human judge-review labels (\xA76g validation pass)";static examples=["<%= config.bin %> evals validate-judge <target-id> # auto-detect installed CLIs","<%= config.bin %> evals validate-judge <target-id> --engine codex --model gpt-5.5","<%= config.bin %> evals validate-judge <target-id> --model claude-sonnet-5 --json","<%= config.bin %> evals validate-judge <target-id> --limit 5 # quick sample"];static args={targetId:p.string({description:"Eval target ID",required:!0})};static flags={model:s.string({description:"Second reviewer model. Default: resolved from your installed tooling (codex: your ~/.codex/config.toml model; claude: a tier that differs from the original judge)."}),engine:s.string({description:"CLI that runs the second reviewer. `auto` prefers your installed codex for a cross-vendor reviewer \u2014 two Claude models share a training lineage, so their agreement overstates independence.",options:["auto","claude","codex"],default:"auto"}),limit:s.integer({description:"Sample only the N most-recently-annotated runs (cost control / smoke)"}),"skip-reviewed":s.boolean({description:"Only re-grade assertions this reviewer model has not ruled on yet",default:!1}),parallel:s.integer({description:"Concurrent re-grades",default:3}),timeout:s.integer({description:"Timeout per re-grade in ms",default:12e4}),json:s.boolean({description:"Emit the full report as JSON"})};async run(){let{args:h,flags:i}=await this.parse(n);this.requireAuth();let a;try{a=await m({targetId:h.targetId,engineFlag:i.engine,modelFlag:i.model,limit:i.limit,parallel:i.parallel,timeoutMs:i.timeout,skipReviewed:i["skip-reviewed"],log:r(t=>this.log(t),"log"),warn:r(t=>this.warn(t),"warn")})}catch(t){throw t instanceof c&&this.error(t.message),t}let{report:e,judgeModels:l,reviewerModel:d}=a;if(l.length===1&&l[0]===d&&this.warn(`Second reviewer resolved to the SAME model as the original judge (${l[0]}) \u2014 this measures self-consistency, not independent review.`),i.json){this.log(JSON.stringify({target:a.targetName,judgeA:l,judgeB:d,persisted:a.persisted,alreadyReviewed:a.alreadyReviewed,...e},null,2));return}if(e.pairs===0&&a.alreadyReviewed>0){this.log(`All ${a.alreadyReviewed} labeled assertion(s) already reviewed by ${d} \u2014 nothing new to grade.`);return}if(this.log(""),this.log(`Second reviewer: ${d} \u2014 scored against ${e.pairs} human-labeled assertion(s)`+(e.skipped?` (${e.skipped} skipped)`:"")),this.log(""),this.log(` reviewer vs HUMAN: ${o(e.bVsHuman.overall.rate)} overall \xB7 ${o(e.bVsHuman.onJudgePass.rate)} where judge passed \xB7 ${o(e.bVsHuman.onJudgeFail.rate)} where judge failed`),this.log(` original judge vs human: ${o(e.aVsHuman.overall.rate)} (your approve rate)`),this.log(` reviewer vs judge: ${o(e.bVsA.overall.rate)} (inter-model, not truth)`),this.log(` Cohen's \u03BA (reviewer vs human): ${e.kappaBvsHuman==null?"undefined (degenerate marginals)":e.kappaBvsHuman.toFixed(2)}`),this.log(""),e.suggestedAnchorWeight!=null&&this.log(` suggested \xA76g anchor weight if used as a passive label source: ${e.suggestedAnchorWeight.toFixed(2)} (misclassification attenuation (2\xB7acc\u22121)\xB2; validated:false either way)`),e.disagreements.length>0){this.log(""),this.log(` ${e.disagreements.length} disagreement(s) with your labels \u2014 flagged on the Review tab:`);for(let t of e.disagreements.slice(0,10))this.log(` run ${t.runId.slice(0,8)} #${t.expectationIndex}: human=${t.humanPassed?"pass":"fail"} reviewer=${t.judgeBPassed?"pass":"fail"} \u2014 ${t.text.slice(0,80)}`);e.disagreements.length>10&&this.log(` \u2026 and ${e.disagreements.length-10} more (use --json for all)`)}}};export{g as default};
|
package/dist/lib/agent-loop.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c,d,e}from"../chunk-
|
|
1
|
+
import{a,b,c,d,e}from"../chunk-QYKLIRMY.js";import"../chunk-TIOKY2IA.js";import"../chunk-35ST2BX7.js";import"../chunk-MYPDODRJ.js";import"../chunk-O23Z6LTP.js";import"../chunk-PRDYXO6Z.js";import"../chunk-QFVGUZDO.js";import"../chunk-KG3TW5QB.js";import"../chunk-W2MOTOQE.js";import"../chunk-WNAYTVCO.js";import"../chunk-ARZJNTMV.js";import"../chunk-6QKI756T.js";import"../chunk-RSKQRVPJ.js";import"../chunk-DO3KHK2X.js";import"../chunk-UF6US7VU.js";import"../chunk-MSTF7A7D.js";import"../chunk-4HZWSFSY.js";import"../chunk-LVOLDUFC.js";import"../chunk-5RBLIVBF.js";import"../chunk-ZL4MZMUK.js";import"../chunk-N7TGRFVZ.js";import"../chunk-KZ65LEUH.js";import"../chunk-GTOEWPIM.js";import"../chunk-6V4WM73L.js";import"../chunk-JUF4JLZQ.js";import"../chunk-TDD7IOV2.js";import"../chunk-7DFNBRLY.js";import"../chunk-MODXEGCU.js";import"../chunk-GQRCSR2A.js";import"../chunk-UUMW6X2W.js";import"../chunk-BSXUH5FD.js";import"../chunk-6UEFCMR2.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-CHA5SX4Y.js";import"../chunk-WQ2BJQYB.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-KG3TW5QB.js";import"../../chunk-BSXUH5FD.js";import"../../chunk-6UEFCMR2.js";import"../../chunk-YDRZDBHG.js";import"../../chunk-S6B2R2LL.js";import"../../chunk-K25S2MGU.js";import"../../chunk-WQ2BJQYB.js";import"../../chunk-3BVHPQZW.js";import"../../chunk-SMGSLY5N.js";import"../../chunk-PJ3WHQ67.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import"../../chunk-ZLLER3HP.js";export{a as processJudgeReview};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c}from"../chunk-
|
|
1
|
+
import{a,b,c,d}from"../chunk-BSXUH5FD.js";import"../chunk-6UEFCMR2.js";import"../chunk-YDRZDBHG.js";import"../chunk-S6B2R2LL.js";import"../chunk-K25S2MGU.js";import"../chunk-WQ2BJQYB.js";import"../chunk-3BVHPQZW.js";import"../chunk-SMGSLY5N.js";import"../chunk-PJ3WHQ67.js";import"../chunk-N7ODX6PO.js";import"../chunk-H2ORWIEY.js";import"../chunk-O6C2UGBY.js";import"../chunk-ZLLER3HP.js";export{c as JudgeReviewSkip,b as llmExpectationIndexes,a as reviewMatchesReviewer,d as runJudgeReviewPass};
|
package/oclif.manifest.json
CHANGED
|
@@ -405,88 +405,6 @@
|
|
|
405
405
|
"setup.js"
|
|
406
406
|
]
|
|
407
407
|
},
|
|
408
|
-
"antigravity:backfill": {
|
|
409
|
-
"aliases": [],
|
|
410
|
-
"args": {
|
|
411
|
-
"conversationId": {
|
|
412
|
-
"description": "Antigravity conversationId (UUID) — the directory name under ~/.gemini/antigravity-ide/brain/",
|
|
413
|
-
"name": "conversationId",
|
|
414
|
-
"required": true
|
|
415
|
-
}
|
|
416
|
-
},
|
|
417
|
-
"description": "Backfill historical events from an Antigravity conversation transcript into a RuleMetric session. Useful for seeding a conversation that existed before hooks were installed — the live hook intentionally only captures forward.",
|
|
418
|
-
"examples": [
|
|
419
|
-
"<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --session 158db9aa-68ee-43fc-961b-f3a180578694",
|
|
420
|
-
"<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --dry-run"
|
|
421
|
-
],
|
|
422
|
-
"flags": {
|
|
423
|
-
"api-url": {
|
|
424
|
-
"description": "Override API base URL",
|
|
425
|
-
"env": "RULEMETRIC_API_URL",
|
|
426
|
-
"hidden": true,
|
|
427
|
-
"name": "api-url",
|
|
428
|
-
"hasDynamicHelp": false,
|
|
429
|
-
"multiple": false,
|
|
430
|
-
"type": "option"
|
|
431
|
-
},
|
|
432
|
-
"session": {
|
|
433
|
-
"char": "s",
|
|
434
|
-
"description": "RuleMetric session UUID to POST events into. If omitted, resolved from the hook tmp file ($TMPDIR/rulemetric/<conversationId>) — fails if neither is present.",
|
|
435
|
-
"name": "session",
|
|
436
|
-
"hasDynamicHelp": false,
|
|
437
|
-
"multiple": false,
|
|
438
|
-
"type": "option"
|
|
439
|
-
},
|
|
440
|
-
"transcript": {
|
|
441
|
-
"description": "Path to the transcript.jsonl file. Defaults to ~/.gemini/antigravity-ide/brain/<conversationId>/.system_generated/logs/transcript.jsonl",
|
|
442
|
-
"name": "transcript",
|
|
443
|
-
"hasDynamicHelp": false,
|
|
444
|
-
"multiple": false,
|
|
445
|
-
"type": "option"
|
|
446
|
-
},
|
|
447
|
-
"after-step": {
|
|
448
|
-
"description": "Only emit records with step_index > this value. Default: read from existing state file, else -1 (everything).",
|
|
449
|
-
"name": "after-step",
|
|
450
|
-
"hasDynamicHelp": false,
|
|
451
|
-
"multiple": false,
|
|
452
|
-
"type": "option"
|
|
453
|
-
},
|
|
454
|
-
"limit": {
|
|
455
|
-
"description": "Cap on the number of records to process this run.",
|
|
456
|
-
"name": "limit",
|
|
457
|
-
"hasDynamicHelp": false,
|
|
458
|
-
"multiple": false,
|
|
459
|
-
"type": "option"
|
|
460
|
-
},
|
|
461
|
-
"dry-run": {
|
|
462
|
-
"description": "Classify records without POSTing. Reports counts only.",
|
|
463
|
-
"name": "dry-run",
|
|
464
|
-
"allowNo": false,
|
|
465
|
-
"type": "boolean"
|
|
466
|
-
},
|
|
467
|
-
"save-state": {
|
|
468
|
-
"description": "After backfill, persist the highest step_index so the live hook resumes from there.",
|
|
469
|
-
"name": "save-state",
|
|
470
|
-
"allowNo": true,
|
|
471
|
-
"type": "boolean"
|
|
472
|
-
}
|
|
473
|
-
},
|
|
474
|
-
"hasDynamicHelp": false,
|
|
475
|
-
"hiddenAliases": [],
|
|
476
|
-
"id": "antigravity:backfill",
|
|
477
|
-
"pluginAlias": "@rulemetric/cli",
|
|
478
|
-
"pluginName": "@rulemetric/cli",
|
|
479
|
-
"pluginType": "core",
|
|
480
|
-
"strict": true,
|
|
481
|
-
"enableJsonFlag": false,
|
|
482
|
-
"isESM": true,
|
|
483
|
-
"relativePath": [
|
|
484
|
-
"dist",
|
|
485
|
-
"commands",
|
|
486
|
-
"antigravity",
|
|
487
|
-
"backfill.js"
|
|
488
|
-
]
|
|
489
|
-
},
|
|
490
408
|
"auth:create-key": {
|
|
491
409
|
"aliases": [],
|
|
492
410
|
"args": {},
|
|
@@ -928,6 +846,88 @@
|
|
|
928
846
|
"run.js"
|
|
929
847
|
]
|
|
930
848
|
},
|
|
849
|
+
"antigravity:backfill": {
|
|
850
|
+
"aliases": [],
|
|
851
|
+
"args": {
|
|
852
|
+
"conversationId": {
|
|
853
|
+
"description": "Antigravity conversationId (UUID) — the directory name under ~/.gemini/antigravity-ide/brain/",
|
|
854
|
+
"name": "conversationId",
|
|
855
|
+
"required": true
|
|
856
|
+
}
|
|
857
|
+
},
|
|
858
|
+
"description": "Backfill historical events from an Antigravity conversation transcript into a RuleMetric session. Useful for seeding a conversation that existed before hooks were installed — the live hook intentionally only captures forward.",
|
|
859
|
+
"examples": [
|
|
860
|
+
"<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --session 158db9aa-68ee-43fc-961b-f3a180578694",
|
|
861
|
+
"<%= config.bin %> antigravity backfill 26fd215d-f43b-4ea5-b177-dc2c0e696f97 --dry-run"
|
|
862
|
+
],
|
|
863
|
+
"flags": {
|
|
864
|
+
"api-url": {
|
|
865
|
+
"description": "Override API base URL",
|
|
866
|
+
"env": "RULEMETRIC_API_URL",
|
|
867
|
+
"hidden": true,
|
|
868
|
+
"name": "api-url",
|
|
869
|
+
"hasDynamicHelp": false,
|
|
870
|
+
"multiple": false,
|
|
871
|
+
"type": "option"
|
|
872
|
+
},
|
|
873
|
+
"session": {
|
|
874
|
+
"char": "s",
|
|
875
|
+
"description": "RuleMetric session UUID to POST events into. If omitted, resolved from the hook tmp file ($TMPDIR/rulemetric/<conversationId>) — fails if neither is present.",
|
|
876
|
+
"name": "session",
|
|
877
|
+
"hasDynamicHelp": false,
|
|
878
|
+
"multiple": false,
|
|
879
|
+
"type": "option"
|
|
880
|
+
},
|
|
881
|
+
"transcript": {
|
|
882
|
+
"description": "Path to the transcript.jsonl file. Defaults to ~/.gemini/antigravity-ide/brain/<conversationId>/.system_generated/logs/transcript.jsonl",
|
|
883
|
+
"name": "transcript",
|
|
884
|
+
"hasDynamicHelp": false,
|
|
885
|
+
"multiple": false,
|
|
886
|
+
"type": "option"
|
|
887
|
+
},
|
|
888
|
+
"after-step": {
|
|
889
|
+
"description": "Only emit records with step_index > this value. Default: read from existing state file, else -1 (everything).",
|
|
890
|
+
"name": "after-step",
|
|
891
|
+
"hasDynamicHelp": false,
|
|
892
|
+
"multiple": false,
|
|
893
|
+
"type": "option"
|
|
894
|
+
},
|
|
895
|
+
"limit": {
|
|
896
|
+
"description": "Cap on the number of records to process this run.",
|
|
897
|
+
"name": "limit",
|
|
898
|
+
"hasDynamicHelp": false,
|
|
899
|
+
"multiple": false,
|
|
900
|
+
"type": "option"
|
|
901
|
+
},
|
|
902
|
+
"dry-run": {
|
|
903
|
+
"description": "Classify records without POSTing. Reports counts only.",
|
|
904
|
+
"name": "dry-run",
|
|
905
|
+
"allowNo": false,
|
|
906
|
+
"type": "boolean"
|
|
907
|
+
},
|
|
908
|
+
"save-state": {
|
|
909
|
+
"description": "After backfill, persist the highest step_index so the live hook resumes from there.",
|
|
910
|
+
"name": "save-state",
|
|
911
|
+
"allowNo": true,
|
|
912
|
+
"type": "boolean"
|
|
913
|
+
}
|
|
914
|
+
},
|
|
915
|
+
"hasDynamicHelp": false,
|
|
916
|
+
"hiddenAliases": [],
|
|
917
|
+
"id": "antigravity:backfill",
|
|
918
|
+
"pluginAlias": "@rulemetric/cli",
|
|
919
|
+
"pluginName": "@rulemetric/cli",
|
|
920
|
+
"pluginType": "core",
|
|
921
|
+
"strict": true,
|
|
922
|
+
"enableJsonFlag": false,
|
|
923
|
+
"isESM": true,
|
|
924
|
+
"relativePath": [
|
|
925
|
+
"dist",
|
|
926
|
+
"commands",
|
|
927
|
+
"antigravity",
|
|
928
|
+
"backfill.js"
|
|
929
|
+
]
|
|
930
|
+
},
|
|
931
931
|
"eval-targets:create": {
|
|
932
932
|
"aliases": [],
|
|
933
933
|
"args": {},
|
|
@@ -4323,5 +4323,5 @@
|
|
|
4323
4323
|
]
|
|
4324
4324
|
}
|
|
4325
4325
|
},
|
|
4326
|
-
"version": "0.7.
|
|
4326
|
+
"version": "0.7.26"
|
|
4327
4327
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulemetric/cli",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.26",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -74,9 +74,9 @@
|
|
|
74
74
|
"postgres": "^3.4.0",
|
|
75
75
|
"react": "^19.0.0",
|
|
76
76
|
"zod": "^3.24.0",
|
|
77
|
-
"@rulemetric/hooks": "0.7.
|
|
78
|
-
"@rulemetric/
|
|
79
|
-
"@rulemetric/
|
|
77
|
+
"@rulemetric/hooks": "0.7.26",
|
|
78
|
+
"@rulemetric/proxy": "0.7.26",
|
|
79
|
+
"@rulemetric/skills-registry": "0.7.26"
|
|
80
80
|
},
|
|
81
81
|
"//deps": "Worker (`evals agent`) runtime deps: the @opentelemetry/* set, @sentry/node, and postgres are STATIC top-level imports loaded at worker startup (via bundled @rulemetric/{telemetry,db}) — they MUST be runtime deps or a clean `npm i -g` crashes with ModuleLoadError. `better-sqlite3` is optionalDependencies (below): it's a native module, lazy `require()`'d inside a try/catch (packages/session providers cursor/opencode) so a missing binary degrades gracefully — making it a hard dep would break `npm i -g` on platforms without prebuilt binaries. `resend` stays a devDep: lazy `await import('resend')` guarded by RESEND_API_KEY, which end-user workers never set (provider falls back to noop). Classification verified via import-graph scan of dist/commands/evals/agent.js + per-importer static/lazy check.",
|
|
82
82
|
"optionalDependencies": {
|
|
@@ -94,8 +94,8 @@
|
|
|
94
94
|
"tsx": "^4.19.0",
|
|
95
95
|
"typescript": "^5.9.0",
|
|
96
96
|
"vitest": "^3.2.0",
|
|
97
|
-
"@rulemetric/core": "0.0.1",
|
|
98
97
|
"@rulemetric/api": "0.0.1",
|
|
98
|
+
"@rulemetric/core": "0.0.1",
|
|
99
99
|
"@rulemetric/db": "0.0.1",
|
|
100
100
|
"@rulemetric/effectiveness": "0.0.1",
|
|
101
101
|
"@rulemetric/email": "0.0.1",
|
package/dist/chunk-LLBGUOG3.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{b as D,c as j}from"./chunk-6UEFCMR2.js";import{a as G}from"./chunk-YDRZDBHG.js";import{c as J}from"./chunk-K25S2MGU.js";import{b as L,c as R,d as N,e as z}from"./chunk-3BVHPQZW.js";import{e as p,g as B}from"./chunk-H2ORWIEY.js";import{a as h}from"./chunk-ZLLER3HP.js";var C=100;function W(i){let a=new Set;return i.forEach((g,s)=>{g.method!=="deterministic"&&a.add(s)}),a}h(W,"llmExpectationIndexes");var f=class extends Error{constructor(g,s){super(s);this.reason=g}static{h(this,"JudgeReviewSkip")}};async function ae(i){let a=i.log??(()=>{}),g=i.warn??a,s=i.targetId,K=i.parallel??3,U=i.timeoutMs??12e4,w=await p(`/api/eval-targets/${s}`),u=(await p(`/api/eval-targets/${s}/annotations?limit=500`)).filter(e=>e.judgeMethod==="llm");if(u.length===0)throw new f("no-labels",`No human labels on "${w.name}" yet \u2014 label some runs in the Review tab first.`);let n;try{n=z({engineFlag:i.engineFlag??"auto",modelFlag:i.modelFlag,availability:N(),judgeModels:[...new Set(u.map(e=>e.judgeModel).filter(e=>e!=null))]})}catch(e){throw new f("no-engine",e instanceof Error?e.message:String(e))}a(`Reviewer: ${n.rationale}`);let c=0,b=u;if(i.skipReviewed&&n.model){let e=await p(`/api/eval-targets/${s}/judge-reviews`),t=new Set(e.filter(r=>r.reviewerModel===n.model).map(r=>`${r.evalRunId}:${r.expectationIndex}`));if(b=u.filter(r=>!t.has(`${r.evalRunId}:${r.expectationIndex}`)),c=u.length-b.length,b.length===0)return{targetName:w.name,reviewer:n,judgeModels:[],reviewerModel:n.model,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:0}}let y=new Map;for(let e=0;;e+=C){let t=await p(`/api/eval-targets/${s}/runs?limit=${C}&offset=${e}`);for(let r of t)y.set(r.id,r);if(t.length<C)break}let q=await p(`/api/eval-targets/${s}/evals`),X=new Map(q.map(e=>[e.id,e.prompt])),o=new Map;for(let e of b){let t=o.get(e.evalRunId)??[];t.push(e),o.set(e.evalRunId,t)}let P=new Map;for(let e of u){let t=P.get(e.evalRunId)??new Set;t.add(e.expectationIndex),P.set(e.evalRunId,t)}let v=0;for(let[e]of[...o]){let t=y.get(e)?.grading?.expectations;if(!t)continue;let r=W(t),l=P.get(e)??new Set;[...r].every(x=>l.has(x))||(o.delete(e),v+=1)}if(v>0&&a(`${v} partially-labeled run(s) deferred \u2014 a run is re-graded once, after all its assertions are labeled.`),o.size===0)return{targetName:w.name,reviewer:n,judgeModels:[],reviewerModel:n.model??`${n.engine} (CLI default)`,report:j([],0),persisted:0,alreadyReviewed:c,deferredPartialRuns:v};if(n.engine==="codex"&&n.model){let e=await J({prompt:"Reply with exactly: OK",model:n.model,timeout:6e4});if(e.error&&L(e.error)){if(n.model===R)throw new f("preflight-failed",`Codex fallback model ${R} is also unrunnable: ${e.error}`);g(`Your codex model "${n.model}" can't run on the installed CLI/account \u2014 falling back to ${R}.`),n={...n,model:R}}else if(e.error)throw new f("preflight-failed",`Codex preflight failed: ${e.error}`)}let S=[...o.values()].flat(),Y=[...new Set(S.map(e=>e.judgeModel).filter(e=>e!=null))];a(`Re-grading ${o.size} annotated run(s) / ${S.length} assertion(s) on "${w.name}"`+(c?` (${c} already reviewed \u2014 skipped)`:"")+".");let I=[],$=0,k=new Set,m=i.limit!=null?[...o.entries()].slice(0,Math.max(1,i.limit)):[...o.entries()];m.length<o.size&&a(`Sampling ${m.length} of ${o.size} annotated runs (limit).`);let H=0,Q=h(async()=>{for(;;){let e=H++;if(e>=m.length)return;let[t,r]=m[e],l=y.get(t),x=l?.grading?.expectations,F=l?.outputs?.transcript,O=l?X.get(l.evalId):void 0;if(!l||!x?.length||!F||!O){$+=r.length,g(`run ${t.slice(0,8)}: missing output/expectations/prompt \u2014 skipped`);continue}let M=await G({prompt:O,output:F,expectations:x.map(d=>d.text),model:n.model??void 0,timeout:U,engine:n.engine});M.judge_model&&k.add(M.judge_model);for(let d of r){let T=x[d.expectationIndex],E=M.expectations[d.expectationIndex];if(!T||!E||T.text!==d.expectationText){$+=1;continue}if(M.judge_model==null&&E.method==="llm"){$+=1;continue}I.push({runId:t,expectationIndex:d.expectationIndex,text:d.expectationText,humanPassed:D(d.verdict,d.judgePassed),judgeAPassed:d.judgePassed,judgeBPassed:E.passed})}a(` ${e+1}/${m.length} re-graded`)}},"workerFn");await Promise.all(Array.from({length:Math.max(1,Math.min(K,m.length))},Q));let V=j(I,$),_=[...k][0]??n.model??`${n.engine} (CLI default)`,A=0;if(I.length>0)try{A=(await B(`/api/eval-targets/${s}/judge-reviews`,{reviewerEngine:n.engine,reviewerModel:_,verdicts:I.map(t=>({evalRunId:t.runId,expectationIndex:t.expectationIndex,expectationText:t.text,judgePassed:t.judgeAPassed,humanPassed:t.humanPassed,reviewerPassed:t.judgeBPassed}))})).upserted,a(`Persisted ${A} reviewer verdict(s) \u2014 disagreements appear on the Review tab.`)}catch(e){g(`Could not persist reviewer verdicts: ${e instanceof Error?e.message:e}`)}return{targetName:w.name,reviewer:n,judgeModels:Y,reviewerModel:_,report:V,persisted:A,alreadyReviewed:c,deferredPartialRuns:v}}h(ae,"runJudgeReviewPass");export{W as a,f as b,ae as c};
|