openpond 0.2.54 → 0.2.56

package/dist/chunks/chunk-4EEH7MEC.js

@@ -1,2 +0,0 @@
-import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
-import{Fa as O}from"./chunk-2JO2KWT3.js";import{n as _,o as k,r as q,u as U,w as j,x as L}from"./chunk-WV2APYWG.js";import{Y as C,h as b,i as x}from"./chunk-PJM2G7SG.js";import{i as T,m as u,n as M}from"./chunk-NNF6KCHD.js";import{b as R}from"./chunk-ES2ISFJC.js";import{a as I}from"./chunk-FTQLQJQV.js";var y=R.object({sessionId:R.string().trim().min(1).max(500),turnId:R.string().trim().min(1).max(500)}).strict();async function Q(e){U(e.manifest,e.taskset,e.catalog),L(e.taskset,e.metricSource);let n=e.manifest.profileEvaluation,o=e.catalog.definitions.find(a=>a.id===n.definitionId),i=new Map(e.taskset.tasks.map(a=>[a.id,a])),p=[],t=[];for(let a of e.manifest.population){e.signal?.throwIfAborted();let l=i.get(a.taskId),s=await e.loadCompletedMember?.(a)??null;if(s){let{receipt:h,grade:g}=s,{contentHash:H,...v}=b.parse(g);if(!k(h)||H!==u(v)||h.id!==a.receiptId||h.runManifest.id!==e.manifest.id||h.runManifest.contentHash!==e.manifest.contentHash||h.taskId!==a.taskId||h.seed!==a.seed||h.graderEvidenceRefs.length!==1||h.graderEvidenceRefs[0]?.contentHash!==H||g.taskHash!==u(l)||g.graderSetHash!==u(e.taskset.graders)||h.metadata.score!==g.score||h.metadata.passed!==g.passed)throw new Error(`Retained Profile evaluation member ${a.receiptId} differs from its admitted evidence.`);t.push(g),p.push(h);continue}let d=await e.execute({task:C(l),seed:a.seed,source:n,...e.signal?{signal:e.signal}:{}});e.signal?.throwIfAborted();let c=await x({task:l,evidence:d.evidence,graders:e.taskset.graders,...e.modelJudge?{modelJudge:e.modelJudge}:{},...e.customVerifier?{customVerifier:e.customVerifier}:{},...e.signal?{signal:e.signal}:{}}),w=T.parse(await e.saveGrade(c));if(w.contentHash!==c.contentHash)throw new Error("Stored Profile evaluation grade differs from its graded evidence.");let E=d.failureClass??c.failureClass,P=_({schemaVersion:"openpond.attemptReceipt.v1",id:a.receiptId,runManifest:{id:e.manifest.id,contentHash:e.manifest.contentHash},taskId:a.taskId,seed:a.seed,terminal:d.terminal,failureClass:E,outputHash:d.evidence.infrastructureError?null:u(d.evidence.output),traceHash:d.traceHash,artifactRefs:d.artifactRefs,graderEvidenceRefs:[w],startedAt:d.startedAt,completedAt:d.completedAt,latencyMs:d.latencyMs,costUsd:d.costUsd,metadata:{gradingRole:"evaluation",score:c.score,passed:c.passed,rewardEligible:c.rewardEligible,sourceRevision:n.sourceRevision,definitionHash:n.definitionHash,environmentHash:n.environmentHash,...d.retainedEvidenceRef?{retainedEvidenceRef:y.parse(d.retainedEvidenceRef)}:{}}});await e.saveReceipt(P),t.push(c),p.push(P)}let r=await j({manifest:e.manifest,taskset:e.taskset,receipts:p,...e.metricSource!==void 0?{source:e.metricSource}:{},...e.signal?{signal:e.signal}:{}},e.metricExecutor),f=t.filter(a=>a.passed).length/t.length;return{receipts:p,grades:t,metric:r,passRate:f,passed:f>=o.criterion.minimumPassRate&&(!o.criterion.requireComplete||p.every(a=>a.terminal))}}I(Q,"executeProfileEvaluationRun");function $(e,n,o){if(n==="agent_action"){let i=e.find(p=>p.name==="workspace_action_result"&&p.action==="profile_workflow_action")?.output;if(typeof i!="string")throw new Error("Profile Agent action evaluation did not produce a released action result.");return i}if(o==="completed"){let t=[...e].reverse().find(f=>f.name==="turn.completed")?.data?.providerResponse;if(typeof t?.requestId!="string"||!t.requestId||typeof t.contentHash!="string")throw new Error("Profile evaluation has no retained final provider response.");let r=e.filter(f=>f.name==="assistant.delta"&&f.data?.providerRequestId===t.requestId).map(f=>f.output??"").join("");if(u(r)!==t.contentHash)throw new Error("Profile evaluation final provider response differs from its retained trace.");return r}return e.filter(i=>i.name==="assistant.delta"&&typeof i.output=="string").map(i=>i.output??"").join("")}I($,"profileEvaluationOutput");async function V(e){let{store:n,manifest:o,receipt:i}=e;if(!i.metadata.retainedEvidenceRef)return null;let p=y.parse(i.metadata.retainedEvidenceRef),[t,r]=await Promise.all([n.getSession(p.sessionId),n.getTurn(p.turnId)]),f=t?.metadata?.profileEvaluationRun,a=o.profileEvaluation;if(!t||!r||r.sessionId!==t.id||!a||!t.currentProfile||u(t.currentProfile)!==u(e.profileRef)||u(f)!==u({id:o.id,contentHash:o.contentHash})||t.metadata?.taskId!==i.taskId||t.metadata?.seed!==i.seed||!r.completedAt||r.completedAt!==i.completedAt||r.startedAt!==i.startedAt||!r.harnessSnapshot||r.harnessSnapshot.harnessRelease.id!==a.harnessRelease.id||r.harnessSnapshot.harnessRelease.contentHash!==a.harnessRelease.contentHash||o.policy.kind!=="model"||!r.modelRef||r.modelRef.providerId!==o.policy.model.provider||r.modelRef.modelId!==o.policy.model.model)throw new Error("Retained Profile policy turn differs from its admitted case.");let l=await n.runtimeEventsForTurn(r.id);if(l.some(g=>g.turnId!==r.id)||u(l)!==i.traceHash)throw new Error("Retained Profile policy trace differs from its immutable receipt.");let s=$(l,a.target.kind,r.status);if(i.outputHash!==null&&u({text:s})!==i.outputHash)throw new Error("Retained Profile output differs from its immutable receipt.");let d=i.artifactRefs.flatMap(g=>{let H=l.filter(m=>(m.action==="work_output_save"||m.action==="sandbox_save_output")&&m.status==="completed").flatMap(m=>{let z=m.data&&typeof m.data=="object"?m.data:{},A=O.safeParse(z.outputRef);return A.success?[A.data]:[]}).filter(m=>m.sourceTaskId===t.id&&m.sourceTurnId===r.id&&`${t.id}/${m.id}/${m.revision}/${m.title}`===g.id&&m.sha256===g.contentHash&&m.sizeBytes===g.sizeBytes&&m.contentType===g.mediaType);if(H.length!==1)throw new Error("The case output has no unique sealed saved-file identity.");let v=H[0];if(v.location.kind!=="managed")return[];let S=`/api/work/outputs/${encodeURIComponent(v.location.fileId)}`;if(v.location.downloadPath!==S)throw new Error("The saved output has no canonical managed download route.");return[{id:v.id,title:v.title,revision:v.revision,contentType:v.contentType,sizeBytes:v.sizeBytes,sha256:v.sha256,downloadPath:S}]}),c=await n.listModelUsageRecords({sessionId:t.id,turnId:r.id,limit:1e4}),w=e.eventAfterId?l.findIndex(g=>g.id===e.eventAfterId):-1;if(e.eventAfterId&&w<0)throw new Error("Profile trace cursor is unavailable in this case.");let E=w+1,P=Math.max(1,Math.min(250,e.eventLimit??100)),h=l.slice(E,E+P);return{ref:p,prompt:r.prompt,output:i.outputHash===null?null:{text:s},partialOutput:i.outputHash===null&&s?{text:s}:null,error:r.error,events:h,eventCount:l.length,nextEventCursor:E+h.length<l.length?h.at(-1).id:null,files:d,usage:c,usageCoverage:c.length===1e4?"partial":"complete"}}I(V,"readProfileEvaluationPolicyEvidence");var oe=R.object({runId:R.string().trim().min(1).max(240),receiptId:R.string().trim().min(1).max(500).optional(),eventAfterId:R.string().trim().min(1).max(500).optional(),eventLimit:R.number().int().min(1).max(250).optional(),manifest:q.optional()}).strict().refine(e=>e.receiptId||!e.eventAfterId&&!e.eventLimit,"Select a case before requesting its trace page.").refine(e=>!e.manifest||e.receiptId,"Interrupted-run inspection requires one exact case.");async function de(e){let n=await e.store.getProfileEvaluationRun(e.runId);if(n&&u(n.profileRef)!==u(e.profileRef))throw new Error("Profile evaluation run is unavailable in the selected Profile.");let o=n?.manifest??e.manifest;if(!o||!n&&!e.receiptId)throw new Error("Profile evaluation run is unavailable in the selected Profile.");if(n){if(e.manifest&&e.manifest.contentHash!==n.manifest.contentHash)throw new Error("Profile evaluation run differs from its pinned manifest.")}else if(M(o,"Interrupted Profile evaluation manifest"),o.id!==e.runId||o.profileEvaluation?.profileId!==e.profileRef.profileId||e.profileRef.source==="openpond_git"&&o.metadata.profileRepositoryId!==e.profileRef.repositoryId)throw new Error("Profile evaluation run is unavailable in the selected Profile.");let i=o.population.map((t,r)=>({member:t,index:r})).filter(({member:t})=>!e.receiptId||t.receiptId===e.receiptId);if(e.receiptId&&i.length!==1)throw new Error("Profile evaluation case is unavailable in the selected run.");let p=await Promise.all(i.map(async({member:t,index:r})=>{let f=n?null:await e.store.getProfileEvaluationReceipt(t.receiptId),a=n?.receiptRefs[r]??(f?{id:f.id,contentHash:f.contentHash}:null),l=n?.gradeRefs[r]??f?.graderEvidenceRefs[0];if(!a||!l)throw new Error("Profile evaluation run has incomplete case evidence.");let[s,d]=await Promise.all([e.store.getProfileEvaluationReceipt(a.id),e.store.getProfileEvaluationGrade(l.contentHash)]);if(!s||!n&&!k(s)||s.contentHash!==a.contentHash||s.id!==t.receiptId||s.taskId!==t.taskId||s.seed!==t.seed||s.runManifest.id!==o.id||s.runManifest.contentHash!==o.contentHash||!d||d.contentHash!==l.contentHash||!s.graderEvidenceRefs.some(c=>c.id===l.id&&c.contentHash===l.contentHash))throw new Error("Profile evaluation case evidence differs from its retained run.");return{taskId:t.taskId,seed:t.seed,receiptId:s.id,receiptHash:s.contentHash,traceHash:s.traceHash,outputHash:s.outputHash,score:d.score,passed:d.passed,gradingStatus:d.gradingStatus,failureClass:d.failureClass,artifactCount:s.artifactRefs.length,latencyMs:s.latencyMs,costUsd:s.costUsd,terminal:s.terminal,feedback:d.components.map(c=>({graderId:c.graderId,graderVersion:c.graderVersion,score:c.score,passed:c.passed,feedback:c.feedback})),...e.receiptId?{receipt:s,evidence:await V({store:e.store,profileRef:e.profileRef,manifest:o,receipt:s,eventAfterId:e.eventAfterId,eventLimit:e.eventLimit})}:{}}}));return{runId:o.id,sourceRevision:o.profileEvaluation?.sourceRevision??null,cases:p}}I(de,"inspectProfileEvaluationRun");export{Q as a,$ as b,oe as c,de as d};