openpond 0.2.58 → 0.2.60

package/dist/chunks/chunk-3VUATFZD.js

@@ -1,2 +0,0 @@
-import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
-import{a as M,b as x,c as A}from"./chunk-4EPSHPZL.js";import{Ea as q,Fa as U,Ia as T,Ja as $,La as j,Oa as D,Ra as O,Sa as K,Ta as z,Ua as _,Va as G,la as P}from"./chunk-25MBPJVY.js";import{e as w,l as c}from"./chunk-WN3V6KDN.js";import{Hb as R,Kb as H,Ob as C,v as k}from"./chunk-Y4OWBBDD.js";import{B as I,Cb as h,Db as b,Ua as v,Xa as E,Ya as y,db as S}from"./chunk-ACYFMLWG.js";import{b as a}from"./chunk-ES2ISFJC.js";import{a as p}from"./chunk-FTQLQJQV.js";var L=a.string().regex(/^[a-f0-9]{64}$/),J=a.object({request:q,maximumCostUsd:a.number().finite().positive().max(1e4),graders:a.array(D).min(1).max(100),admissionRequestHash:L.optional(),sourceExperimentId:a.string().trim().min(1).max(200).optional()}).strict(),X=J.extend({configurationHash:L}).strict(),F=U.extend({configuration:X}).strict();function g(r){return r.policy.kind==="hosted_harness"?{...r,population:r.population.map(({receiptId:e,...t})=>t)}:r}p(g,"experimentConfigurationRequest");function V(r){let e=J.parse(r);return c({...e,request:g(e.request)})}p(V,"experimentRunConfigurationHash");async function Q(r){let e=F.parse(r),{configuration:t,...s}=e,n=await $(s),{configurationHash:o,...i}=t;if(o!==V(i)||c(g(t.request))!==c(g(n.request))||t.maximumCostUsd!==n.manifest.limits.maximumSpendUsd)throw new Error("Experiment configuration differs from its retained run.");if(n.manifest.metadata.experimentDefinition===null){let d=K.parse({definition:null,configurationHash:t.configurationHash,maximumCostUsd:t.maximumCostUsd,graders:t.graders,...t.admissionRequestHash?{admissionRequestHash:t.admissionRequestHash}:{},...t.sourceExperimentId?{sourceExperimentId:t.sourceExperimentId}:{}});if(n.manifest.metadata.experimentConfigurationHash!==c(d))throw new Error("Experiment configuration differs from its admitted manifest.")}return{...n,configuration:t}}p(Q,"verifyExperimentRunDetails");var W=a.object({teamId:a.string().trim().min(1).max(240),items:a.array(a.object({name:a.string().trim().min(1).max(500),source:S,ready:a.boolean(),reason:a.string().min(1).max(2e3).nullable()}).strict()).max(8),nextCursor:a.string().min(1).max(8192).nullable()}).strict(),u=a.string().trim().min(1).max(200),Y=class extends Error{static{p(this,"OpenPondExperimentError")}constructor(r,e,t){super(t),this.status=r,this.code=e,this.name="OpenPondExperimentError"}status;code},we=class{static{p(this,"OpenPondExperimentsClient")}constructor(r){this.options=r;let e=new URL(r.baseUrl);if(!["http:","https:"].includes(e.protocol)||e.username||e.password||e.search||e.hash||!r.apiKey.trim()||!r.teamId.trim())throw new Error("A clean API origin and workspace credentials are required.");this.baseUrl=e.toString().replace(/\/+$/,"")}options;baseUrl;async harnessSources(r={}){let e=r.cursor===void 0?void 0:a.string().min(1).max(8192).parse(r.cursor),t=new URLSearchParams(e?{cursor:e}:{}),s=W.parse(await this.request(`/harness-sources?${t}`,"GET",void 0,r.signal));if(s.teamId!==this.options.teamId||new Set(s.items.map(n=>c(n.source))).size!==s.items.length||s.nextCursor!==null&&s.nextCursor===e||s.items.some(n=>n.ready!==(n.reason===null)))throw new Error("Harness catalog differs from its workspace, readiness or continuation.");return s}async prepareHarness(r,e){let t=z.parse(r),s=_.parse(await this.request("/harness-setup","POST",t,e));T(s.request,s.manifest);let n=s.request.policy;if(s.request.teamId!==this.options.teamId||s.request.operationId!==t.operationId||n.kind!=="hosted_harness"||n.profileRepositoryId!==t.profileRepositoryId||n.source.definitionId!==t.definitionId||n.modelId!==t.modelId||n.reasoningEffort!==t.reasoningEffort||s.maximumCostUsd!==t.maximumCostUsd||s.manifest.limits.maximumSpendUsd!==t.maximumCostUsd)throw new Error("Harness preparation differs from its requested scope or configuration.");return s}async run(r,e){let t=G.parse(r);if(t.request.teamId!==this.options.teamId)throw new Error("Experiment workspace mismatch.");let s=await this.details(await this.request("","POST",t,e));if(c(g(s.request))!==c(g(t.request))||s.configuration.maximumCostUsd!==t.maximumCostUsd||s.configuration.sourceExperimentId!==t.sourceExperimentId||s.configuration.admissionRequestHash!==c(t))throw new Error("Experiment admission differs from its submitted configuration.");if(t.graders&&c(t.graders)!==c(s.configuration.graders.map(n=>({id:n.id,version:n.version,contentHash:n.contentHash,mappings:n.mappings??[]}))))throw new Error("Experiment admission differs from its selected graders or mappings.");return s}async get(r,e={}){return this.details(await this.request(`/${encodeURIComponent(u.parse(r))}`,"GET",void 0,e.signal),r)}async list(r={},e){let t=O.parse(r),s=new URLSearchParams(Object.entries(t).filter(([,i])=>i!==void 0).map(([i,d])=>[i,String(d)])),n=a.object({items:a.array(a.unknown()).max(100),nextCursor:u.nullable()}).strict().parse(await this.request(`?${s}`,"GET",void 0,e)),o=await Promise.all(n.items.map(i=>this.details(i)));if(o.length>t.limit||new Set(o.map(i=>i.summary.id)).size!==o.length||n.nextCursor!==null&&n.nextCursor!==o.at(-1)?.summary.id||o.some(i=>t.projectId&&i.request.project?.id!==t.projectId||t.datasetHash&&i.request.taskset.contentHash!==t.datasetHash||t.status&&i.summary.status!==t.status))throw new Error("Experiment page differs from its selected scope.");return{items:o,nextCursor:n.nextCursor}}async result(r,e){let t=a.object({executionId:u,passId:u.nullable(),manifest:a.unknown(),result:a.unknown()}).strict().parse(await this.request(`/${encodeURIComponent(u.parse(r))}/result`,"GET",void 0,e)),s=h(t);if(t.executionId!==r||s.manifest.teamId!==this.options.teamId||s.manifest.lineage?.execution.id!==r||s.manifest.lineage.scoringPassId!==t.passId)throw new Error("Experiment result identity mismatch.");return{...s,executionId:r,passId:t.passId}}async cancel(r,e){return this.details(await this.request(`/${encodeURIComponent(u.parse(r))}/cancel`,"POST",{},e),r)}async duplicate(r,e,t){let s=await this.details(await this.request(`/${encodeURIComponent(u.parse(r))}/duplicate`,"POST",{operationId:u.parse(e)},t));if(s.request.operationId!==e||s.configuration.sourceExperimentId!==r||s.summary.id===r)throw new Error("Duplicate Experiment admission differs from its source or operation.");return s}async score(r,e){let t=M.parse(r),s=this.pass(await this.request(`/${encodeURIComponent(t.execution.id)}/scoring-passes`,"POST",t,e));if(c(s.request)!==c(t))throw new Error("Scoring admission differs from its submitted inputs.");return s}async scoringPass(r,e){return this.pass(await this.request(`/scoring-passes/${encodeURIComponent(u.parse(r))}`,"GET",void 0,e),r)}async scoringPasses(r,e={}){let t=new URLSearchParams(e.afterId?{afterId:u.parse(e.afterId)}:{});e.executionKind&&t.set("executionKind",a.literal("recorded_evidence").parse(e.executionKind));let s=a.object({items:a.array(a.unknown()).max(30),nextCursor:u.nullable()}).strict().parse(await this.request(`/${encodeURIComponent(u.parse(r))}/scoring-passes?${t}`,"GET",void 0,e.signal)),n=s.items.map(o=>this.pass(o));if(n.some(o=>o.request.execution.id!==r||o.request.executionKind!==e.executionKind)||new Set(n.map(o=>o.id)).size!==n.length)throw new Error("Scoring pass page differs from its execution ownership.");return{items:n,nextCursor:s.nextCursor}}async cancelScoringPass(r,e){return this.pass(await this.request(`/scoring-passes/${encodeURIComponent(u.parse(r))}/cancel`,"POST",{},e),r)}async scoringResult(r,e){let t=a.object({executionId:u,passId:u,manifest:a.unknown(),result:a.unknown()}).strict().parse(await this.request(`/scoring-passes/${encodeURIComponent(u.parse(r))}/result`,"GET",void 0,e)),s=h(t);if(t.passId!==r||s.manifest.teamId!==this.options.teamId||s.manifest.id!==r||s.manifest.lineage?.execution.id!==t.executionId||s.manifest.lineage.scoringPassId!==r)throw new Error("Scoring result identity mismatch.");return{...s,executionId:t.executionId,passId:r}}async compare(r,e,t){let s=p(i=>i.startsWith("score_")?this.scoringResult(i,t):this.result(i,t),"read"),[n,o]=await Promise.all([s(r),s(e)]);return{baseline:n,candidate:o,comparison:b(n,o)}}async details(r,e){let t=await Q(r);if(t.summary.teamId!==this.options.teamId||e&&t.summary.id!==e)throw new Error("Experiment identity mismatch.");return t}pass(r,e){let t=x(r);if(t.teamId!==this.options.teamId||e&&t.id!==e)throw new Error("Scoring pass identity mismatch.");return t}async request(r,e,t,s){let n=await(this.options.fetch??fetch)(`${this.baseUrl}/v1/experiments${r}`,{method:e,signal:s,redirect:"error",headers:{Authorization:`Bearer ${this.options.apiKey}`,"X-OpenPond-Team-Id":this.options.teamId,"Content-Type":"application/json",Accept:"application/json"},...t===void 0?{}:{body:JSON.stringify(t)}}),o=await n.json();if(!n.ok){let i=a.object({code:a.string(),message:a.string()}).safeParse(o);throw new Y(n.status,i.success?i.data.code:"experiment_request_failed",i.success?i.data.message:"Experiment request failed.")}return o}},l=a.string().trim().min(1).max(500),N=a.string().regex(/^[a-f0-9]{64}$/),Z=a.object({schemaVersion:a.literal("openpond.experimentCaseInspection.v1"),teamId:l,executionId:l,manifestHash:N,receiptId:l,taskId:l,seed:a.string().min(1),input:a.record(a.string(),a.unknown()).nullable(),output:a.unknown().nullable(),error:a.string().nullable(),events:a.array(a.record(a.string(),a.unknown())).max(25),nextEventCursor:l.nullable(),eventCount:a.number().int().nonnegative()}).strict(),ee=Z.extend({contentHash:N}).strict(),Ie=class{static{p(this,"OpenPondExperimentInspectionClient")}constructor(r){this.options=r;let e=new URL(r.baseUrl);if(!["http:","https:"].includes(e.protocol)||e.username||e.password||e.search||e.hash||!r.apiKey.trim()||!r.teamId.trim())throw new Error("A clean API origin and workspace credentials are required.");this.baseUrl=e.toString().replace(/\/+$/,"")}options;baseUrl;async case(r,e,t={}){let s=new URLSearchParams(t.afterId?{afterId:l.parse(t.afterId)}:{}),n=await(this.options.fetch??fetch)(`${this.baseUrl}/v1/experiments/${encodeURIComponent(l.parse(r))}/cases/${encodeURIComponent(l.parse(e))}?${s}`,{signal:t.signal,redirect:"error",headers:{Authorization:`Bearer ${this.options.apiKey}`,"X-OpenPond-Team-Id":this.options.teamId,Accept:"application/json"}}),o=await n.json();if(!n.ok)throw new Error(a.object({message:a.string()}).parse(o).message);let i=ee.parse(o),{contentHash:d,...m}=i;if(c(m)!==d||i.teamId!==this.options.teamId||i.executionId!==r||i.receiptId!==e||t.manifestHash&&i.manifestHash!==t.manifestHash||i.events.length>i.eventCount)throw new Error("Case inspection differs from its requested workspace, execution or retained content.");return i}};function re(r){return JSON.stringify(r.release?["reward",r.release.id,r.release.revision,r.release.contentHash,r.feedbackKey]:["dataset",r.id,r.version,r.contentHash,r.feedbackKey])}p(re,"datasetGraderIdentity");function te(r){let e=new Map;for(let t of r){let s=re(t),n=e.get(s);n?n.aliases.push(t):e.set(s,{key:s,grader:t,aliases:[t]})}return[...e.values()]}p(te,"groupDatasetGraders");function ve(r){let e=te(r),t=new Map;for(let s of e){let n=t.get(s.grader.feedbackKey);if(n&&n!==s.key)throw new Error(`Different selected grader releases use feedback key "${s.grader.feedbackKey}". Select one, or publish graders with distinct feedback keys.`);t.set(s.grader.feedbackKey,s.key);let o=JSON.stringify(s.grader.mappings??[]);if(s.aliases.some(i=>JSON.stringify(i.mappings??[])!==o))throw new Error(`Aliases of grader "${s.grader.feedbackKey}" have different field mappings. Select one alias and review its mappings.`)}return e.map(s=>s.grader)}p(ve,"canonicalDatasetGraderSelection");function be(r){let e=v(r.binding),t=P(r.packageValue),s=t.taskset;if(s.id!==e.dataset.id||s.revision!==e.dataset.revision||s.contentHash!==e.dataset.contentHash||t.contentHash!==e.packageHash||c(s.environment)!==e.environmentHash||c(t.files.filter(n=>n.asset.visibility!=="policy").map(n=>n.asset))!==e.privateDatasetClosureHash)throw new Error("The selected Dataset package, environment or protected closure changed.");if(e.population.length>500)throw new Error("Select at most 500 exact Dataset cases before starting this bounded Profile evaluation.");for(let n of e.population){let o=s.tasks.find(i=>i.id===n.taskId);if(!o||o.split!==e.split)throw new Error("The selected ordered population is outside its exact Dataset split.");if(n.fixtureId!==null)throw new Error("This recorded cutoff needs an admitted replay fixture before executing the Profile target.")}if(e.gradingSource){if(!r.actualGrading)throw new Error("The actual selected scoring owner is required for this recipe.");let n=x(r.actualGrading.pass),o=h(a.object({manifest:a.unknown(),result:a.unknown()}).strict().parse(r.actualGrading.evidence));if(n.status!=="completed"||n.id!==e.gradingSource.pass.id||n.contentHash!==e.gradingSource.pass.contentHash||o.manifest.id!==e.gradingSource.evidence.id||o.result.contentHash!==e.gradingSource.evidence.contentHash||o.manifest.lineage?.scoringPassId!==n.id||c(o.manifest.dataset)!==c(e.dataset)||c(o.manifest.evaluators)!==c(e.evaluators)||c(e.fieldMappings)!==c(n.graders.map(m=>({graderId:m.id,fields:m.mappings??[]}))))throw new Error("The selected scoring owner changed its exact Dataset, measurements or mappings.");let i=o.manifest.population,d=e.recordedOrigin?e.recordedOrigin.members.map(m=>({caseId:m.caseId,seed:m.seed,fixtureId:m.fixtureId})):e.population.map(m=>({caseId:m.taskId,seed:m.seed,fixtureId:m.fixtureId}));if(i.length!==d.length||i.some((m,B)=>c(m)!==c(d[B])))throw new Error("The selected scoring population differs from the exact ordered original cases and cutoffs.")}else{if(e.fieldMappings.some(i=>!s.graders.some(d=>d.id===i.graderId)))throw new Error("The field mappings name a grader outside the selected release.");for(let i of e.fieldMappings)j.parse(i.fields);let n=e.evaluators.map(i=>{let d=r.graders.find(m=>m.feedbackKey===i.feedbackKey&&c(m.release??{id:m.id,revision:m.version,contentHash:m.contentHash})===c(i.release));if(!d)throw new Error("Select an exact released Dataset grader for this external recipe.");return d});if(e.fieldMappings.some(i=>!n.some(d=>d.id===i.graderId)))throw new Error("Field mappings refer to an unselected grader.");let o=n.map(i=>({release:i.release??{id:i.id,revision:i.version,contentHash:i.contentHash},feedbackKey:i.feedbackKey,configurationHash:c({graderHash:i.contentHash,mappings:e.fieldMappings.find(d=>d.graderId===i.id)?.fields??[]}),output:"score",categories:[]}));if(c(o)!==c(e.evaluators))throw new Error("The external recipe's graders or mappings differ from the current exact released grading settings.");for(let i of r.graders)if(!s.graders.some(d=>d.id===i.id&&d.version===i.version&&c(d)===i.contentHash))throw new Error("External grader pins differ from the actual Dataset implementation.")}for(let n of r.graders)if(!s.graders.some(o=>o.id===n.id&&o.version===n.version&&c(o)===n.contentHash))throw new Error("Dataset-native grader pins differ from their actual implementation.");return t}p(be,"verifyExternalDatasetPackage");var f=a.string().trim().min(1).max(240),se=a.object({profileRef:a.object({source:a.enum(["local","github","openpond_git"]),repositoryId:f,profileId:f}).strict(),sourceRevision:a.string().min(1).max(500),harnessRelease:w,catalogHash:a.string().regex(/^[a-f0-9]{64}$/).nullable(),definitions:a.array(E).max(1e3),suites:a.array(y).max(100),runs:a.array(a.object({profileRef:a.object({source:a.enum(["local","github","openpond_git"]),repositoryId:f,profileId:f}).strict(),manifest:k,metric:I,gradeRefs:a.array(w),receiptRefs:a.array(w),passRate:a.number().min(0).max(1),passed:a.boolean(),completedAt:a.iso.datetime(),contentHash:a.string().regex(/^[a-f0-9]{64}$/)}).strict()).max(1e3),comparisons:a.array(R).max(1e3),suiteRuns:a.array(C).max(1e3),reports:a.array(H).max(1e3)}).strict(),ae=a.object({teamId:f,profileRepositoryId:f,trainingProjectId:f.optional()}).strict(),He=class{static{p(this,"OpenPondProfileEvaluationDiscoveryClient")}constructor(r){this.options=r;let e=new URL(r.baseUrl);if(!["http:","https:"].includes(e.protocol)||e.username||e.password||e.search||e.hash||!r.apiKey.trim()||!r.teamId.trim())throw new Error("Profile discovery requires a scoped HTTP endpoint and owner credential.")}options;async read(r,e){let t=ae.parse({...r,teamId:this.options.teamId}),{response:s,value:n}=await A(this.options.fetch??fetch,`${this.options.baseUrl.replace(/\/$/,"")}/v1/profile-evaluation-discovery`,{method:"POST",redirect:"error",headers:{Authorization:`Bearer ${this.options.apiKey}`,"X-OpenPond-Team-Id":this.options.teamId,"Content-Type":"application/json"},body:JSON.stringify(t),signal:e},(i,d,m)=>new Error(m));if(!s.ok)throw new Error("The released Profile evaluation owner is unavailable to this account and workspace.");let o=se.parse(n);if(o.profileRef.repositoryId!==t.profileRepositoryId)throw new Error("Profile discovery returned another source owner.");return o}};export{Y as a,we as b,Ie as c,ve as d,be as e,He as f};

package/dist/chunks/chunk-3Z47M3YY.js

@@ -1,2 +0,0 @@
-import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
-import{a as H,d as Q,f as X}from"./chunk-TRQQNFMO.js";import{a as ee}from"./chunk-FEAKTSRS.js";import{a as re}from"./chunk-SDAUS5PY.js";import{d as ae,i as oe,j as ie}from"./chunk-O72BSKSO.js";import{d as te}from"./chunk-FUOEIDMA.js";import{I as Y,K as Z}from"./chunk-VRIEKBBH.js";import{o as se,s as b}from"./chunk-WJ43MJ3F.js";import{Ib as K,ca as P,la as g,xa as U}from"./chunk-25MBPJVY.js";import{w as G,x as F,z as T}from"./chunk-Y4OWBBDD.js";import{a as j,f as q}from"./chunk-FFRFSDBM.js";import{g as L}from"./chunk-VYLGI3IH.js";import{Ta as N,Ua as y,Wa as W,ab as $,e as _}from"./chunk-ACYFMLWG.js";import{b as R,c as v,d as J,l as S,m as n}from"./chunk-KGNCKIFP.js";import{b as h}from"./chunk-ES2ISFJC.js";import{a as d}from"./chunk-FTQLQJQV.js";import{randomUUID as V,createHash as fe}from"node:crypto";import{mkdtemp as ue,mkdir as pe,writeFile as D,rm as me}from"node:fs/promises";import he from"node:os";import E from"node:path";var ne=d(async e=>{let a=g(e.packageValue);if(e.grader.runtime!=="sandbox_process")throw new Error("Select the sandbox process runtime for a local file grader.");if(!a.taskset.graders.some(r=>n(r)===n(e.grader)))throw new Error("The private grader differs from its package.");if(!a.taskset.tasks.some(r=>n(r)===n(e.task)))throw new Error("The private case differs from its package.");await e.authorize(),e.signal?.throwIfAborted();let i=await ue(E.join(he.tmpdir(),"profile-private-grade-")),t=V();try{for(let f of a.files.filter(l=>l.asset.visibility!=="policy")){let l=f.asset.path;if(E.isAbsolute(l)||l.split(/[\\/]/).some(u=>!u||u==="."||u===".."))throw new Error("Private grader asset path invalid.");let m=E.join(i,"package",l);await pe(E.dirname(m),{recursive:!0,mode:448}),await D(m,P(f),{mode:256})}await D(E.join(i,"request.json"),JSON.stringify({task:e.task,attempt:e.evidence,input:e.task.input,output:e.evidence.output,artifacts:"artifacts"in e.evidence?e.evidence.artifacts:[],expectedOutput:e.task.expectedOutput}),{mode:256});let r=`import fs from 'node:fs/promises'; const module=await import(${JSON.stringify("./package/"+e.grader.verifierRef.path)}); const result=await module[${JSON.stringify(e.grader.exportName??"verify")}](JSON.parse(await fs.readFile('request.json','utf8'))); console.log(JSON.stringify(result));`;await D(E.join(i,"runner.mjs"),r,{mode:256});let s=await ee({authorize:d(async()=>(await e.authorize(),e.signal?.throwIfAborted(),{candidateId:t,candidateRevision:1,ownerId:t,sessionId:t,turnId:t,sourceRoot:i,writablePaths:[]}),"authorize")})({candidateId:t,sessionId:t,turnId:t,expectedRevision:1,command:"/runtime/node --max-old-space-size=512 runner.mjs",timeoutMs:e.grader.timeoutMs,maxOutputBytes:7e4,signal:e.signal});if(await e.authorize(),e.signal?.throwIfAborted(),s.code!==0||s.timedOut||s.stdoutTruncated||s.stderrTruncated)throw new Error("Private grading sandbox did not complete.");return H.parse(JSON.parse(s.stdout))}finally{await me(i,{recursive:!0,force:!0})}},"executeLocalProfileVerifier");function Me(e){return async a=>{if(!a.manifest)throw new Error("The hosted grader requires its admitted manifest.");await a.authorize(),a.signal?.throwIfAborted();let i="artifacts"in a.evidence&&Array.isArray(a.evidence.artifacts)?a.evidence.artifacts.map(({reference:l,title:m,contentType:u})=>({reference:l,title:m,contentType:u})):[],t="caseOwner"in a.evidence?a.evidence.caseOwner:null;if(!t?.sessionId||!t.turnId)throw new Error("The private grader has no sealed case owner.");let r=Buffer.from(JSON.stringify({manifest:a.manifest,packageValue:a.packageValue,graderId:a.grader.id,taskId:a.task.id,evidence:{...a.evidence,artifacts:i}}));if(r.length>2e7)throw new Error("The private grading package exceeds its transport limit.");let o={callId:V(),sessionId:t.sessionId,turnId:t.turnId,sha256:fe("sha256").update(r).digest("hex"),sizeBytes:r.length,timeoutMs:a.grader.timeoutMs},s=d((l,m=12e4)=>e.request(Z.parse({contractVersion:Y,requestId:V(),operation:"profile-evaluations/grade",params:l}),m),"request");for(let l=0;l<r.length;l+=98304)await a.authorize(),a.signal?.throwIfAborted(),await s({...o,action:"upload",offset:l,contentsBase64:r.subarray(l,l+98304).toString("base64")});let f=await s({...o,action:"execute"},a.grader.timeoutMs+12e4);return await a.authorize(),a.signal?.throwIfAborted(),H.parse(f)}}d(Me,"createHostedProfileVerifier");async function ge(e,a){let i=g(e);for(let r of i.taskset.graders){if(r.kind==="model_judge"&&(!a||r.calibrationStatus!=="passed"))throw new Error("This Profile requires its current calibrated judge and bounded private provider owner.");let o="verifierRef"in r?r.verifierRef:"rubricRef"in r?r.rubricRef:null;o&&A(i,o)}let t=i.taskset.metrics?.customAggregator;if(t){let r=i.files.find(o=>o.asset.path===t.module);if(!r||r.asset.visibility==="policy"||S(P(r))!==t.contentHash)throw new Error("The authored aggregate metric is not its exact private release.");new TextDecoder("utf8",{fatal:!0}).decode(P(r))}}d(ge,"preflightProfilePrivateGrading");function A(e,a){let i=e.files.find(t=>t.asset.id===a.id);if(!i||i.asset.visibility==="policy"||n(i.asset)!==n(a))throw new Error("The exact private grader asset is unavailable.");return new TextDecoder("utf8",{fatal:!0}).decode(P(i))}d(A,"readPrivate");async function $e(e){let a=g(e.packageValue);if(a.contentHash!==e.manifest.packageHash||a.taskset.contentHash!==e.manifest.tasksetRelease.contentHash)throw new Error("The private grading package changed its frozen Run identity.");await e.authorize(),await ge(a,!!e.executeBudgetedJudge),await e.authorize();let i=d(async({grader:o,task:s,evidence:f})=>{let l=e.signal;await e.authorize(),l?.throwIfAborted();let m={manifest:e.manifest,packageValue:a,grader:o,task:s,evidence:f,signal:l,authorize:e.authorize},u=e.executeSandboxVerifier?await e.executeSandboxVerifier(m):o.runtime==="sandbox_process"?await ne(m):await Q({source:A(a,o.verifierRef),exportName:o.exportName,runtime:o.runtime,timeoutMs:o.timeoutMs,signal:l,value:{task:s,attempt:f,input:s.input,output:f.output,artifacts:"artifacts"in f?f.artifacts:[],expectedOutput:s.expectedOutput,infrastructureError:f.infrastructureError??null}});return await e.authorize(),l?.throwIfAborted(),{score:u.score,passed:u.passed,rewardEligible:o.rewardEligible,failureClass:null,feedback:[u.feedback],visibleEvidenceRefs:[],privilegedEvidenceRefs:u.evidenceRefs}},"customVerifier"),t=a.taskset.metrics?.customAggregator,r=t?new TextDecoder("utf8",{fatal:!0}).decode(P(a.files.find(o=>o.asset.path===t.module))):void 0;return{authorize:e.authorize,customVerifier:i,...e.executeBudgetedJudge?{modelJudge:L({readRubric:d(async o=>{await e.authorize();let s=A(a,o);return await e.authorize(),s},"readRubric"),executeBudgeted:d(async(o,s)=>{await e.authorize(),s?.throwIfAborted();let f=await e.executeBudgetedJudge(o,s);return await e.authorize(),s?.throwIfAborted(),f},"executeBudgeted")})}:{},...r!==void 0?{metricSource:r,metricExecutor:d(async o=>{await e.authorize(),o.signal?.throwIfAborted();let s=await X(o);return await e.authorize(),o.signal?.throwIfAborted(),s},"metricExecutor")}:{}}}d($e,"createProfilePrivateGradingOwner");import{promises as ve}from"node:fs";import ke from"node:path";async function at(e){let a=e.definition.tasksetRelease,i=await te(e.store,e.harnessRelease);if(!i)throw new Error("Evaluation Profile Harness release is unavailable.");let t=await Re({...e,runtime:i});if(t)return t;let r=await e.store.listTasksetRevisions(e.profileId);for(let o of r){if(a.id!==o.id&&a.id!==`taskset-release-${o.id}-r${o.revision}`)continue;let s=await we(e.storeDir,o);if(s.taskset.id===a.id&&s.taskset.contentHash===a.contentHash)return s}throw new Error(`Released Taskset ${a.id} is unavailable in Profile ${e.profileId}.`)}d(at,"loadLocalProfileEvaluationTaskset");async function we(e,a){let i=a.metadata.importedPackageHash;if(typeof i=="string"){let o=await ie(e,i);if(o.taskset.id!==a.id||o.taskset.revision!==a.revision)throw new Error("Imported evaluation Taskset differs from its local Profile revision.");return o}if(U(a)!==a.contentHash)throw new Error("Authored evaluation Taskset differs from its immutable revision.");let t=K({taskset:a,adapterId:ae(a)}),r=await re({storeDir:e,taskset:a,content:{schemaVersion:"openpond.tasksetPackage.v1",taskset:t.tasksetRelease,environment:t.environmentRelease,verifierSet:t.verifierSetRelease}});return await oe(e,r),r}d(we,"loadCandidate");async function Re(e){let a=e.definition.tasksetRelease,i=e.runtime;if(i.release.harnessRelease.id!==e.harnessRelease.id||i.release.harnessRelease.contentHash!==e.harnessRelease.contentHash)throw new Error("Evaluation Taskset runtime differs from its requested release.");let t=`evals/tasksets/${a.contentHash}.json`,r=i.release.harnessRelease.files.find(o=>o.path===t);if(r){if(r.visibility!=="verifier")throw new Error("Evaluation Taskset package is not verifier-private.");let o=await ve.readFile(ke.join(i.release.bundlePath,"source",...t.split("/")));if(S(o)!==r.contentHash)throw new Error("Released evaluation Taskset package bytes differ from their source receipt.");let s=g(JSON.parse(o.toString("utf8")));if(s.taskset.id!==a.id||s.taskset.contentHash!==a.contentHash)throw new Error("Released evaluation Taskset package differs from its definition.");return s}return null}d(Re,"loadReleasedProfileEvaluationTaskset");function C(e){return{id:W(e),label:"Selected Dataset evaluation",description:"Explicit sealed external Dataset evaluation recipe.",target:e.target,tasksetRelease:{id:e.dataset.id,contentHash:e.dataset.contentHash},split:e.split,taskIds:[...new Set(e.population.map(a=>a.taskId))],seeds:[...new Set(e.population.map(a=>a.seed))],criterion:e.criterion}}d(C,"externalProfileEvaluationDefinition");async function lt(e){let a=e.manifest.profileEvaluation,i=a?.externalDatasetBinding?y(a.externalDatasetBinding):null;if(!i)return null;if(!e.resolveExternalDataset)throw new Error("This owner has no authorized external Dataset resolver.");let t=await e.resolveExternalDataset(i,e.selected);await t.authorize();let r=g(t.packageValue),o=await e.loadCatalog({ref:e.selected.profileRef,sourceRevision:e.selected.sourceRevision,harnessRelease:e.selected.harnessRelease});if(n(t.binding)!==n(i)||i.profileId!==e.selected.profileRef.profileId||i.declaredProfileCatalogHash!==o.catalogHash||r.contentHash!==i.packageHash||n(r.taskset)!==n(e.taskset)||i.environmentHash!==n(r.taskset.environment)||i.privateDatasetClosureHash!==n(r.files.filter(l=>l.asset.visibility!=="policy").map(l=>l.asset)))throw new Error("External Dataset admission changed its actual Profile, package or private closure.");let s=C(i),f={schemaVersion:"openpond.profileEvaluations.v1",definitions:[s],suites:[]};if(T(e.manifest,e.taskset,f),!a||a.catalogHash!==n(f)||a.definitionHash!==n(s)||n(a.target)!==n(i.target))throw new Error("External Dataset manifest changed its derived definition.");return await t.authorize(),{...t,packageValue:r,definition:s,catalog:f}}d(lt,"admitProfileExternalDataset");var Pe=h.object({id:R,createdAt:J,definitionId:R,externalDatasetBinding:N.optional(),profileRef:se.optional(),profileSource:h.object({sourceRevision:h.string().min(1).max(500),harnessRelease:h.object({id:R,contentHash:v}).strict()}).strict().optional(),modelRef:q,hostModelConfigurationHash:v.optional(),hostArtifactHash:v.optional(),maximumSpendUsd:h.number().positive().max(1e4).optional(),expectedManifestHash:v.optional(),hostExperiment:h.object({definition:h.object({id:R,revision:h.number().int().positive(),contentHash:v}).strict().nullable(),configurationHash:v,project:h.object({id:R,revision:h.number().int().positive(),contentHash:v,targetId:R.nullable()}).strict().optional()}).strict().optional()}).strict();function vt(e){return async(a,i)=>{let t=Pe.parse(a);if(i?.requireExpectedManifestHash&&!t.expectedManifestHash)throw new Error("Profile evaluation execution requires the reviewed manifest hash.");if(t.profileSource&&!t.profileRef)throw new Error("An exact Profile source requires its accepted Profile reference.");let r=i?.selectedWorkflows??await e.selectedWorkflows(t.profileRef,t.profileSource);if(t.profileRef&&n(t.profileRef)!==n(r.profileRef))throw new Error("Profile preparation did not resolve the explicitly selected Profile.");if(t.profileSource&&(r.sourceRevision!==t.profileSource.sourceRevision||n(r.harnessRelease)!==n(t.profileSource.harnessRelease)))throw new Error("Profile preparation changed its explicitly pinned source.");let o=t.externalDatasetBinding?y(t.externalDatasetBinding):null,s=o?await(e.resolveExternalDataset?e.resolveExternalDataset(o,r):Promise.reject(new Error("This host has not qualified external Dataset Profile execution."))):null;if(s&&(await s.authorize(),n(y(s.binding))!==n(o)))throw new Error("External Dataset admission differs from the actual authorized recipe.");let f=await e.loadCatalog({ref:r.profileRef,sourceRevision:r.sourceRevision,harnessRelease:r.harnessRelease});if(o&&o.declaredProfileCatalogHash!==f.catalogHash)throw new Error("The original Profile verifier catalog changed.");let l=o?C(o):null;if(l&&t.definitionId!==l.id)throw new Error("Use the external recipe's own definition identity.");let m={schemaVersion:"openpond.profileEvaluations.v1",definitions:l?[l]:f.definitions,suites:l?[]:f.suites},u=l??f.definitions.find(c=>c.id===t.definitionId);if(!u)throw new Error(`Evaluation ${t.definitionId} is absent from the selected Profile release.`);let k=u.target,z=k.kind==="workflow"?r.workflows.find(c=>c.binding.workflowId===k.workflowId)?.binding:_.parse({schemaVersion:"openpond.profileComponentBinding.v1",profileId:r.profileRef.profileId,sourceRevision:r.sourceRevision,harnessRelease:r.harnessRelease,target:k});if(!z)throw new Error(`Evaluation workflow ${k.kind==="workflow"?k.workflowId:""} is absent from the selected Profile release.`);let w=g(s?.packageValue??await e.loadTasksetPackage(u,r.profileRef.profileId,r.harnessRelease)),p=w.taskset;if(p.id!==u.tasksetRelease.id||p.contentHash!==u.tasksetRelease.contentHash)throw new Error("Evaluation Taskset package differs from its released definition.");let le=p.environment.kind==="text"&&p.tools.length===0,M=p.environment.kind==="work"&&p.environment.entrypoint==="openpond-work-v1"&&p.tools.every(c=>["work_exec","work_save_output"].includes(c.name));if(!le&&!M)throw new Error("Profile evaluation supports text cases or Work cases with the built-in execution and output tools.");if(t.maximumSpendUsd!==void 0&&M&&!e.boundedWorkComputeAvailable)throw new Error("This host has not admitted budgeted Work compute.");if(p.tasks.some(c=>c.artifactRefs.length>b.maxAttachments||c.artifactRefs.some(x=>x.visibility!=="policy"||x.mediaType!=="application/pdf"||x.sizeBytes>b.maxAttachmentBytes)))throw new Error("Profile evaluation accepts only bounded policy-visible PDF task attachments.");if(!o&&!e.privateGradingPreflight&&(p.graders.some(c=>c.kind==="model_judge"||c.kind==="custom_verifier")||p.metrics?.aggregation==="custom"))throw new Error("Profile evaluation cannot run this Taskset's model judge, custom verifier, or custom metric in the current app-server runtime.");await e.privateGradingPreflight?.(w);let B={adapterId:k.kind==="workflow"?"openpond.profile-workflow":"openpond.profile-component",placement:e.placement,runtimeVersion:"app-server-v1",capabilityReceipt:n({adapterId:k.kind==="workflow"?"openpond.profile-workflow":"openpond.profile-component",packageHash:w.contentHash,connectedAppScopes:p.policy.connectedAppScopes})},ce={...$({catalog:m,definitionId:u.id,profileId:r.profileRef.profileId,sourceRevision:r.sourceRevision,harnessRelease:r.harnessRelease,environmentHash:n({packageHash:w.contentHash,runtimeTarget:B})}),...o?{externalDatasetBinding:o}:{},...e.sourceCandidate?{sourceCandidate:e.sourceCandidate}:{}};if(o&&(o.packageHash!==w.contentHash||o.environmentHash!==n(p.environment)||o.privateDatasetClosureHash!==n(w.files.filter(c=>c.asset.visibility!=="policy").map(c=>c.asset))))throw new Error("External Dataset package/environment/private closure changed.");let de=j.parse({provider:t.modelRef.providerId,model:t.modelRef.modelId,revision:null,artifactHash:t.hostArtifactHash??null,tokenizerRevision:null,chatTemplateHash:null}),O=await e.modelConfigurationHash(t.modelRef,t),I=G({schemaVersion:"openpond.tasksetRunManifest.v1",id:t.id,tasksetRelease:u.tasksetRelease,packageHash:w.contentHash,execution:{kind:"harness",harnessRelease:r.harnessRelease},profileEvaluation:ce,policy:{kind:"model",model:de,configurationHash:O},gradingRole:"evaluation",metricPolicy:F(p),population:o?o.population.map(c=>({...c,receiptId:`evaluation-${n([t.id,c.taskId,c.seed,c.fixtureId]).slice(0,32)}`})):u.taskIds.flatMap(c=>u.seeds.map(x=>({receiptId:`evaluation-${n([t.id,c,x]).slice(0,32)}`,taskId:c,seed:x,fixtureId:null}))),runtimeTarget:B,limits:{maxTurns:128,timeoutMs:p.environment.defaultTimeoutMs,maxOutputBytes:25e7,maximumSpendUsd:t.maximumSpendUsd??null},createdAt:t.createdAt,metadata:{sourceTasksetId:p.metadata.sourceTasksetId??p.id,...r.profileRef.source==="openpond_git"?{profileRepositoryId:r.profileRef.repositoryId}:{},...t.hostExperiment?{experimentDefinition:t.hostExperiment.definition,experimentConfigurationHash:t.hostExperiment.configurationHash,...t.hostExperiment.project?{project:t.hostExperiment.project}:{}}:{}}});if(T(I,p,m),s&&await s.authorize(),t.expectedManifestHash&&I.contentHash!==t.expectedManifestHash)throw new Error("Profile evaluation setup changed since preview; prepare the run again.");return{manifest:I,taskset:p,profileRef:r.profileRef,binding:z,modelRef:t.modelRef,modelConfigurationHash:O}}}d(vt,"createProfileEvaluationRunPreparationService");export{ne as a,Me as b,ge as c,$e as d,at as e,Re as f,lt as g,vt as h};