openpond 0.2.43 → 0.2.45

package/dist/chunks/chunk-E6WT4HWF.js

@@ -1,3 +0,0 @@
-import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
-import{b as E}from"./chunk-MDE3HHLH.js";import{a as u,b as v,c as R,e as P,l as m}from"./chunk-HZP5U2AA.js";import{b as n}from"./chunk-ES2ISFJC.js";import{a as f}from"./chunk-FTQLQJQV.js";var C=n.object({profileId:u,sourceRevision:n.string().trim().min(1).max(240),harnessRelease:P}).strict(),I=n.object({kind:n.enum(["run","suite","comparison"]),id:u,contentHash:v}).strict(),g=n.object({schemaVersion:n.literal("openpond.profileEvaluationReport.v1"),id:u,createdAt:R,testedSources:n.array(C).min(1).max(100),tasksetReleases:n.array(P).min(1).max(100),metricPolicyHashes:n.array(v).min(1).max(100),modelConfigurationHashes:n.array(v).min(1).max(100),evidence:n.array(I).min(1).max(100),summary:n.object({totalChecks:n.number().int().nonnegative(),passedChecks:n.number().int().nonnegative(),failedChecks:n.number().int().nonnegative(),unscoredChecks:n.number().int().nonnegative(),meanScore:n.number().min(0).max(1).nullable()}).strict()}).strict().superRefine((e,i)=>{let{summary:t}=e;t.passedChecks+t.failedChecks+t.unscoredChecks!==t.totalChecks&&i.addIssue({code:"custom",path:["summary"],message:"Report check counts must add up."}),new Set(e.evidence.map(({kind:l,id:a})=>`${l}:${a}`)).size!==e.evidence.length&&i.addIssue({code:"custom",path:["evidence"],message:"Report evidence must be unique."})}),w=g.safeExtend({contentHash:v});function S(e){let i=g.parse(e);return w.parse({...i,contentHash:m(i)})}f(S,"createProfileEvaluationReport");function y(e){let i=w.parse(e),{contentHash:t,...l}=i;if(m(l)!==t)throw new Error("Profile evaluation report content hash differs from its source bytes.");return i}f(y,"assertProfileEvaluationReport");var x=n.object({id:u,runIds:n.array(u).min(2).max(100)}).strict();function z(e){return async i=>{let t=x.parse(i),l=await e.selectedProfile();if(!l)throw new Error("Select a Profile before comparing its evaluation runs.");let a=[];for(let r of t.runIds){let s=await e.store.getProfileEvaluationRun(r);if(!s||m(s.profileRef)!==m(l.ref))throw new Error(`Profile evaluation run ${r} is unavailable in the selected Profile.`);a.push({manifest:s.manifest,result:s.metric})}let o=await e.store.getProfileEvaluationComparison(t.id),c=E({id:t.id,members:a,createdAt:o?.createdAt??new Date().toISOString()});return e.store.saveProfileEvaluationComparison(l.ref,c)}}f(z,"createProfileEvaluationComparisonService");import{promises as d}from"node:fs";import h from"node:path";var j=n.object({id:u,evidenceKind:n.enum(["run","suite","comparison"]),evidenceId:u}).strict();function k(e){return h.join(e,"evals","reports")}f(k,"reportDirectory");async function b(e){let i=await d.lstat(e).catch(t=>{if(t.code==="ENOENT")return null;throw t});if(i&&(!i.isDirectory()||i.isSymbolicLink()))throw new Error("Profile evaluation report directory must be a regular directory.");i||await d.mkdir(e)}f(b,"ensureRegularDirectory");async function K(e){let i,t;if(e.evidenceKind==="run")i=[e.evidenceId],t="";else if(e.evidenceKind==="suite"){let o=await e.store.getProfileEvaluationSuiteRun(e.evidenceId);if(!o||o.profileId!==e.profileRef.profileId)throw new Error("Profile evaluation suite is unavailable.");i=o.members.map(c=>c.runManifest.id),t=o.contentHash}else{let o=await e.store.getProfileEvaluationComparison(e.evidenceId);if(!o)throw new Error("Profile evaluation comparison is unavailable.");i=o.members.map(c=>c.runManifest.id),t=o.contentHash}let l=await Promise.all(i.map(o=>e.store.getProfileEvaluationRun(o)));if(l.some(o=>!o||m(o.profileRef)!==m(e.profileRef)))throw new Error("Profile evaluation report is missing authorized run evidence.");let a=l;if(e.evidenceKind==="run"&&(t=a[0].contentHash),e.evidenceKind!=="run"){let o=e.evidenceKind==="suite"?(await e.store.getProfileEvaluationSuiteRun(e.evidenceId)).members.map(c=>c.runHash):(await e.store.getProfileEvaluationComparison(e.evidenceId)).members.map(c=>c.metricResultHash);for(let[c,r]of a.entries())if(e.evidenceKind==="suite"&&o[c]!==r.contentHash||e.evidenceKind==="comparison"&&o[c]!==r.metric.contentHash)throw new Error("Profile evaluation report evidence differs from retained member runs.")}return{runs:a,primary:{kind:e.evidenceKind,id:e.evidenceId,contentHash:t}}}f(K,"loadRuns");async function O(e){let i=j.parse(e.request),{runs:t,primary:l}=await K({store:e.store,profileRef:e.profileRef,evidenceKind:i.evidenceKind,evidenceId:i.evidenceId}),a=f(r=>[...new Map(r.map(s=>[m(s),s])).values()],"unique"),o=a(t.map(r=>m(r.manifest.metricPolicy))),c=t.map(r=>r.metric.value).filter(r=>r!==null);return S({schemaVersion:"openpond.profileEvaluationReport.v1",id:i.id,createdAt:new Date().toISOString(),testedSources:a(t.map(r=>({profileId:r.manifest.profileEvaluation.profileId,sourceRevision:r.manifest.profileEvaluation.sourceRevision,harnessRelease:r.manifest.profileEvaluation.harnessRelease}))),tasksetReleases:a(t.map(r=>r.manifest.tasksetRelease)),metricPolicyHashes:o,modelConfigurationHashes:a(t.map(r=>r.manifest.policy.kind==="model"?r.manifest.policy.configurationHash:m(r.manifest.policy))),evidence:[l,...t.filter(r=>l.kind!=="run"||r.manifest.id!==l.id).map(r=>({kind:"run",id:r.manifest.id,contentHash:r.contentHash}))],summary:{totalChecks:t.length,passedChecks:t.filter(r=>r.passed).length,failedChecks:t.filter(r=>!r.passed&&r.metric.value!==null).length,unscoredChecks:t.filter(r=>!r.passed&&r.metric.value===null).length,meanScore:o.length===1&&c.length===t.length?c.reduce((r,s)=>r+s,0)/c.length:null}})}f(O,"buildProfileEvaluationReport");function Q(e){return{list:f(async()=>{let l=await e.selectedProfile();if(!l)return[];let a=k(l.sourcePath),o=await d.lstat(a).catch(s=>{if(s.code==="ENOENT")return null;throw s});if(!o)return[];if(!o.isDirectory()||o.isSymbolicLink())throw new Error("Profile evaluation report directory is unsafe.");let c=await d.readdir(a,{withFileTypes:!0});return(await Promise.all(c.filter(s=>s.name.endsWith(".json")).map(async s=>{if(!s.isFile()||s.isSymbolicLink())throw new Error("Profile evaluation report must be a regular file.");let p=y(JSON.parse(await d.readFile(h.join(a,s.name),"utf8")));if(`${p.id}.json`!==s.name||p.testedSources.some(H=>H.profileId!==l.ref.profileId))throw new Error("Profile evaluation report differs from its source location.");return p}))).sort((s,p)=>p.createdAt.localeCompare(s.createdAt))},"list"),save:f(async l=>{let a=await e.selectedProfile();if(!a?.gitBacked)throw new Error("Select a Git-backed Profile before saving an evaluation report.");let o=await O({store:e.store,profileRef:a.ref,request:l}),c=h.join(a.sourcePath,"evals");await b(c);let r=k(a.sourcePath);return await b(r),await d.writeFile(h.join(r,`${o.id}.json`),`${JSON.stringify(o,null,2)}
-`,{flag:"wx"}),o},"save")}}f(Q,"createProfileEvaluationReportService");export{z as a,O as b,Q as c};