openpond 0.2.45 → 0.2.47

package/dist/chunks/chunk-XM26DGML.js

@@ -0,0 +1,3 @@
+import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
+import{ka as P,ma as h,na as E}from"./chunk-WV2APYWG.js";import{b as u,m as f}from"./chunk-NNF6KCHD.js";import{b as d}from"./chunk-ES2ISFJC.js";import{a as c}from"./chunk-FTQLQJQV.js";var S=d.object({id:u,runIds:d.array(u).min(2).max(100)}).strict();function j(r){return async s=>{let i=S.parse(s),l=await r.selectedProfile();if(!l)throw new Error("Select a Profile before comparing its evaluation runs.");let a=[];for(let e of i.runIds){let t=await r.store.getProfileEvaluationRun(e);if(!t||f(t.profileRef)!==f(l.ref))throw new Error(`Profile evaluation run ${e} is unavailable in the selected Profile.`);a.push({manifest:t.manifest,result:t.metric})}let o=await r.store.getProfileEvaluationComparison(i.id),n=P({id:i.id,members:a,createdAt:o?.createdAt??new Date().toISOString()});return r.store.saveProfileEvaluationComparison(l.ref,n)}}c(j,"createProfileEvaluationComparisonService");import{promises as m}from"node:fs";import v from"node:path";var y=d.object({id:u,evidenceKind:d.enum(["run","suite","comparison"]),evidenceId:u}).strict();function w(r){return v.join(r,"evals","reports")}c(w,"reportDirectory");async function R(r){let s=await m.lstat(r).catch(i=>{if(i.code==="ENOENT")return null;throw i});if(s&&(!s.isDirectory()||s.isSymbolicLink()))throw new Error("Profile evaluation report directory must be a regular directory.");s||await m.mkdir(r)}c(R,"ensureRegularDirectory");async function k(r){let s,i;if(r.evidenceKind==="run")s=[r.evidenceId],i="";else if(r.evidenceKind==="suite"){let o=await r.store.getProfileEvaluationSuiteRun(r.evidenceId);if(!o||o.profileId!==r.profileRef.profileId)throw new Error("Profile evaluation suite is unavailable.");s=o.members.map(n=>n.runManifest.id),i=o.contentHash}else{let o=await r.store.getProfileEvaluationComparison(r.evidenceId);if(!o)throw new Error("Profile evaluation comparison is unavailable.");s=o.members.map(n=>n.runManifest.id),i=o.contentHash}let l=await Promise.all(s.map(o=>r.store.getProfileEvaluationRun(o)));if(l.some(o=>!o||f(o.profileRef)!==f(r.profileRef)))throw new Error("Profile evaluation report is missing authorized run evidence.");let a=l;if(r.evidenceKind==="run"&&(i=a[0].contentHash),r.evidenceKind!=="run"){let o=r.evidenceKind==="suite"?(await r.store.getProfileEvaluationSuiteRun(r.evidenceId)).members.map(n=>n.runHash):(await r.store.getProfileEvaluationComparison(r.evidenceId)).members.map(n=>n.metricResultHash);for(let[n,e]of a.entries())if(r.evidenceKind==="suite"&&o[n]!==e.contentHash||r.evidenceKind==="comparison"&&o[n]!==e.metric.contentHash)throw new Error("Profile evaluation report evidence differs from retained member runs.")}return{runs:a,primary:{kind:r.evidenceKind,id:r.evidenceId,contentHash:i}}}c(k,"loadRuns");async function H(r){let s=y.parse(r.request),{runs:i,primary:l}=await k({store:r.store,profileRef:r.profileRef,evidenceKind:s.evidenceKind,evidenceId:s.evidenceId}),a=c(e=>[...new Map(e.map(t=>[f(t),t])).values()],"unique"),o=a(i.map(e=>f(e.manifest.metricPolicy))),n=i.map(e=>e.metric.value).filter(e=>e!==null);return h({schemaVersion:"openpond.profileEvaluationReport.v1",id:s.id,createdAt:new Date().toISOString(),testedSources:a(i.map(e=>({profileId:e.manifest.profileEvaluation.profileId,sourceRevision:e.manifest.profileEvaluation.sourceRevision,harnessRelease:e.manifest.profileEvaluation.harnessRelease}))),tasksetReleases:a(i.map(e=>e.manifest.tasksetRelease)),metricPolicyHashes:o,modelConfigurationHashes:a(i.map(e=>e.manifest.policy.kind==="model"?e.manifest.policy.configurationHash:f(e.manifest.policy))),evidence:[l,...i.filter(e=>l.kind!=="run"||e.manifest.id!==l.id).map(e=>({kind:"run",id:e.manifest.id,contentHash:e.contentHash}))],summary:{totalChecks:i.length,passedChecks:i.filter(e=>e.passed).length,failedChecks:i.filter(e=>!e.passed&&e.metric.value!==null).length,unscoredChecks:i.filter(e=>!e.passed&&e.metric.value===null).length,meanScore:o.length===1&&n.length===i.length?n.reduce((e,t)=>e+t,0)/n.length:null}})}c(H,"buildProfileEvaluationReport");function q(r){return{list:c(async()=>{let l=await r.selectedProfile();if(!l)return[];let a=w(l.sourcePath),o=await m.lstat(a).catch(t=>{if(t.code==="ENOENT")return null;throw t});if(!o)return[];if(!o.isDirectory()||o.isSymbolicLink())throw new Error("Profile evaluation report directory is unsafe.");let n=await m.readdir(a,{withFileTypes:!0});return(await Promise.all(n.filter(t=>t.name.endsWith(".json")).map(async t=>{if(!t.isFile()||t.isSymbolicLink())throw new Error("Profile evaluation report must be a regular file.");let p=E(JSON.parse(await m.readFile(v.join(a,t.name),"utf8")));if(`${p.id}.json`!==t.name||p.testedSources.some(g=>g.profileId!==l.ref.profileId))throw new Error("Profile evaluation report differs from its source location.");return p}))).sort((t,p)=>p.createdAt.localeCompare(t.createdAt))},"list"),save:c(async l=>{let a=await r.selectedProfile();if(!a?.gitBacked)throw new Error("Select a Git-backed Profile before saving an evaluation report.");let o=await H({store:r.store,profileRef:a.ref,request:l}),n=v.join(a.sourcePath,"evals");await R(n);let e=w(a.sourcePath);return await R(e),await m.writeFile(v.join(e,`${o.id}.json`),`${JSON.stringify(o,null,2)}
+`,{flag:"wx"}),o},"save")}}c(q,"createProfileEvaluationReportService");export{j as a,H as b,q as c};

package/dist/chunks/{chunk-J65BMG5Q.js → chunk-XON44M6V.js}

@@ -1,2 +1,2 @@
 import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
-import{g as c}from"./chunk-25GCTLAZ.js";import{ca as f}from"./chunk-QVRNOGAU.js";import{a as n}from"./chunk-FTQLQJQV.js";import{promises as v}from"node:fs";import u from"node:path";async function j(e){let s=await c(e.store,e.harnessRelease);if(!s)throw new Error("Profile evaluation Harness release is unavailable.");return w({...e,runtime:s})}n(j,"profileEvaluationsForRelease");async function w(e){let s=e.runtime,o=s.release.harnessRelease;if(o.id!==e.harnessRelease.id||o.contentHash!==e.harnessRelease.contentHash)throw new Error("Profile evaluation runtime differs from its requested release.");let r=o.metadata.profile;if(!r||typeof r!="object"||r.id!==e.ref.profileId||r.sourceRevision!==e.sourceRevision)throw new Error("Profile evaluation release differs from its accepted source.");let i={profileRef:e.ref,sourceRevision:e.sourceRevision,harnessRelease:e.harnessRelease};if(!o.files.some(a=>a.path==="evals/catalog.json"))return{...i,catalogHash:null,definitions:[],suites:[]};let d=u.join(s.release.bundlePath,"source"),t=n(async a=>o.files.some(g=>g.path===a)?v.readFile(u.join(d,...a.split("/"))):void 0,"read"),[R,p,m]=await Promise.all([t("evals/catalog.json"),t("workflows/catalog.json"),t("workflows/actions.json")]),{catalog:l,catalogHash:h}=f({agentSnapshot:s.release.agentSnapshot,harnessRelease:o,catalogBytes:R,workflowBytes:p,actionBytes:m});return{...i,catalogHash:h,definitions:l.definitions,suites:l.suites}}n(w,"profileEvaluationsForRuntime");export{j as a,w as b};
+import{g as c}from"./chunk-A2GVVXFV.js";import{xa as f}from"./chunk-PJM2G7SG.js";import{a as n}from"./chunk-FTQLQJQV.js";import{promises as v}from"node:fs";import u from"node:path";async function j(e){let s=await c(e.store,e.harnessRelease);if(!s)throw new Error("Profile evaluation Harness release is unavailable.");return w({...e,runtime:s})}n(j,"profileEvaluationsForRelease");async function w(e){let s=e.runtime,o=s.release.harnessRelease;if(o.id!==e.harnessRelease.id||o.contentHash!==e.harnessRelease.contentHash)throw new Error("Profile evaluation runtime differs from its requested release.");let r=o.metadata.profile;if(!r||typeof r!="object"||r.id!==e.ref.profileId||r.sourceRevision!==e.sourceRevision)throw new Error("Profile evaluation release differs from its accepted source.");let i={profileRef:e.ref,sourceRevision:e.sourceRevision,harnessRelease:e.harnessRelease};if(!o.files.some(a=>a.path==="evals/catalog.json"))return{...i,catalogHash:null,definitions:[],suites:[]};let d=u.join(s.release.bundlePath,"source"),t=n(async a=>o.files.some(g=>g.path===a)?v.readFile(u.join(d,...a.split("/"))):void 0,"read"),[R,p,m]=await Promise.all([t("evals/catalog.json"),t("workflows/catalog.json"),t("workflows/actions.json")]),{catalog:l,catalogHash:h}=f({agentSnapshot:s.release.agentSnapshot,harnessRelease:o,catalogBytes:R,workflowBytes:p,actionBytes:m});return{...i,catalogHash:h,definitions:l.definitions,suites:l.suites}}n(w,"profileEvaluationsForRuntime");export{j as a,w as b};