openpond 0.2.36 → 0.2.38

package/dist/chunks/chunk-RELOOVWW.js

@@ -0,0 +1,2 @@
+import { createRequire as __openpondCreateRequire } from "node:module"; var require = __openpondCreateRequire(import.meta.url);
+import{Wa as A,v as C}from"./chunk-JUX77H6K.js";import{a as f,b,c as R,e as H,h as j,l as h,m as z}from"./chunk-HZP5U2AA.js";import{b as e}from"./chunk-ES2ISFJC.js";import{a as d}from"./chunk-FTQLQJQV.js";var F=e.number().finite().nonnegative(),V=e.discriminatedUnion("kind",[e.object({kind:e.literal("model"),modelId:f,configurationHash:b}).strict(),e.object({kind:e.literal("harness"),source:A}).strict(),e.object({kind:e.literal("fixture"),configurationHash:b}).strict()]),U=e.object({release:H.extend({revision:e.union([e.number().int().positive(),e.string().trim().min(1).max(200)])}).strict(),feedbackKey:C,configurationHash:b.optional(),output:e.enum(["boolean","score","category"]),categories:e.array(e.string().trim().min(1).max(120)).max(50)}).strict().superRefine((t,n)=>{t.output==="category"!=t.categories.length>0&&n.addIssue({code:"custom",path:["categories"],message:"Only category feedback declares categories."}),new Set(t.categories).size!==t.categories.length&&n.addIssue({code:"custom",path:["categories"],message:"Feedback categories must be unique."})}),M=e.object({caseId:f,seed:e.string().trim().min(1).max(500),fixtureId:f.nullable()}).strict();function K(t){let n=M.parse(t);return JSON.stringify([n.caseId,n.seed,n.fixtureId])}d(K,"experimentCaseKey");var D=e.object({schemaVersion:e.literal("openpond.experimentManifest.v1"),id:f,name:e.string().trim().min(1).max(500),teamId:f,operationId:f,maximumCostUsd:F.nullable(),dataset:H.extend({revision:e.number().int().positive()}).strict(),target:V,execution:e.object({packageHash:b,runtimeTargetHash:b,metricPolicyHash:b}).strict().optional(),lineage:e.object({definition:H.extend({revision:e.number().int().positive()}).strict().nullable(),execution:H,scoringPassId:f.nullable()}).strict().optional(),evaluators:e.array(U).max(100),population:e.array(M).min(1).max(1e4),createdAt:R}).strict().superRefine((t,n)=>{t.lineage&&t.id!==(t.lineage.scoringPassId??t.lineage.execution.id)&&n.addIssue({code:"custom",path:["lineage"],message:"Experiment identity must match its execution or independent scoring pass."}),new Set(t.population.map(K)).size!==t.population.length&&n.addIssue({code:"custom",path:["population"],message:"Experiment cases must be unique."}),new Set(t.evaluators.map(s=>s.feedbackKey)).size!==t.evaluators.length&&n.addIssue({code:"custom",path:["evaluators"],message:"Experiment feedback keys must be unique."})}),q=D.extend({contentHash:b}).strict(),J=e.object({feedbackKey:C,evaluator:U.shape.release,status:e.enum(["scored","unavailable","failed","pending"]),value:e.union([e.boolean(),e.number().finite(),e.string().max(120)]).nullable(),passed:e.boolean().nullable().optional(),reasoning:e.string().max(2e4).nullable(),evidenceRefs:e.array(j).max(1e3)}).strict(),B=e.object({inputTokens:e.number().int().nonnegative().nullable(),outputTokens:e.number().int().nonnegative().nullable(),totalTokens:e.number().int().nonnegative().nullable(),costUsd:F.nullable(),latencyMs:e.number().finite().nonnegative().nullable()}).strict().superRefine((t,n)=>{t.inputTokens!==null&&t.outputTokens!==null&&t.totalTokens!==null&&t.totalTokens<t.inputTokens+t.outputTokens&&n.addIssue({code:"custom",path:["totalTokens"],message:"Total tokens cannot be less than input plus output."})}),G=e.object({identity:M,status:e.enum(["completed","failed","cancelled","unavailable"]),output:e.unknown().nullable(),error:e.object({code:f,message:e.string().max(2e3)}).strict().nullable(),feedback:e.array(J).max(100),usage:B,traceRef:j.nullable(),startedAt:R.nullable(),completedAt:R.nullable()}).strict(),N=e.object({schemaVersion:e.literal("openpond.experimentResult.v1"),manifest:H,status:e.enum(["completed","failed","cancelled"]),cases:e.array(G).min(1).max(1e4),completedAt:R}).strict(),O=N.extend({contentHash:b}).strict();function L(t,n){q.parse(n),z(n,"Experiment manifest");let s=N.parse(t);if(s.manifest.id!==n.id||s.manifest.contentHash!==n.contentHash)throw new Error("Experiment result differs from its manifest.");let v=new Set(n.population.map(K)),l=new Set,w=new Map(n.evaluators.map(g=>[g.feedbackKey,g]));for(let g of s.cases){let S=K(g.identity);if(!v.has(S)||l.has(S))throw new Error("Experiment result has an unexpected or repeated case.");l.add(S);let I=new Set;for(let i of g.feedback){let a=w.get(i.feedbackKey);if(!a||I.has(i.feedbackKey)||i.evaluator.id!==a.release.id||i.evaluator.contentHash!==a.release.contentHash||i.evaluator.revision!==a.release.revision)throw new Error("Experiment feedback differs from its pinned evaluator.");if(I.add(i.feedbackKey),i.status==="scored"){if(i.value===null||typeof i.value!=={boolean:"boolean",score:"number",category:"string"}[a.output]||a.output==="category"&&!a.categories.includes(i.value)||a.output==="score"&&(i.value<0||i.value>1))throw new Error("Experiment feedback value violates its output contract.")}else if(i.value!==null||i.passed!==null&&i.passed!==void 0)throw new Error("Unscored experiment feedback cannot claim a value or pass decision.")}}if(l.size!==v.size)throw new Error("Experiment result omits admitted cases.");return O.parse({...s,contentHash:h(s)})}d(L,"createExperimentResult");function Q(t){let n=q.parse(t.manifest),s=O.parse(t.result),{contentHash:v,...l}=s;if(h(l)!==v)throw new Error("Experiment result integrity failed.");return L(l,n),{manifest:n,result:s}}d(Q,"verifyExperimentEvidence");function $(t,n){for(let a of[t,n])Q(a);let s=[];if(t.manifest.teamId!==n.manifest.teamId&&s.push("different_workspace"),(t.manifest.dataset.id!==n.manifest.dataset.id||t.manifest.dataset.contentHash!==n.manifest.dataset.contentHash||t.manifest.dataset.revision!==n.manifest.dataset.revision)&&s.push("different_dataset_version"),t.manifest.target.kind!==n.manifest.target.kind&&s.push("different_target_kind"),h(t.manifest.execution??null)!==h(n.manifest.execution??null)&&s.push("different_execution_contract"),t.manifest.target.kind==="harness"&&n.manifest.target.kind==="harness"){let a=t.manifest.target.source,r=n.manifest.target.source;(a.profileId!==r.profileId||h(a.target)!==h(r.target))&&s.push("different_component"),a.environmentHash!==r.environmentHash&&s.push("different_environment"),(a.definitionId!==r.definitionId||a.definitionHash!==r.definitionHash)&&s.push("different_evaluation_definition")}let v=d(a=>[...a.evaluators].sort((r,o)=>r.feedbackKey.localeCompare(o.feedbackKey)),"orderedEvaluators");h(v(t.manifest))!==h(v(n.manifest))&&s.push("different_evaluators");let l=new Map(t.result.cases.map(a=>[K(a.identity),a])),w=new Map(n.result.cases.map(a=>[K(a.identity),a])),g=[...new Set([...l.keys(),...w.keys()])].sort();(l.size!==w.size||g.some(a=>!l.has(a)||!w.has(a)))&&s.push("different_population");let S=s.length===0,I=g.map(a=>{let r=l.get(a)??null,o=w.get(a)??null,T=new Map(r?.feedback.map(c=>[c.feedbackKey,c])??[]),_=new Map(o?.feedback.map(c=>[c.feedbackKey,c])??[]);return{identity:(r??o).identity,baseline:r,candidate:o,feedback:t.manifest.evaluators.map(c=>{let y=T.get(c.feedbackKey),x=_.get(c.feedbackKey),m=S?!r||!o?"missing_case":r.status!=="completed"||o.status!=="completed"||r.error||o.error?"case_not_completed":!y||!x||y.status!=="scored"||x.status!=="scored"?"feedback_unavailable":c.output==="category"?"categorical_feedback":null:"incompatible_experiments",u=d(k=>typeof k?.value=="boolean"?Number(k.value):typeof k?.value=="number"?k.value:null,"numeric"),p=m?null:u(y),E=m?null:u(x),P=d(k=>m?null:typeof k?.value=="boolean"?k.value:k?.passed??null,"pass");return{feedbackKey:c.feedbackKey,eligible:m===null,reason:m,baseline:p,candidate:E,delta:p===null||E===null?null:E-p,baselinePassed:P(y),candidatePassed:P(x)}})}}),i=t.manifest.evaluators.map((a,r)=>{let o=I.map(u=>u.feedback[r]).filter(u=>u.eligible),T=d(u=>o.length?o.reduce((p,E)=>p+E[u],0)/o.length:null,"mean"),_=T("baseline"),c=T("candidate"),y=d(u=>o.length&&o.every(p=>p[u]!==null)?o.reduce((p,E)=>p+Number(E[u]),0)/o.length:null,"passRate"),x=y("baselinePassed"),m=y("candidatePassed");return{feedbackKey:a.feedbackKey,eligibleCount:o.length,excludedCount:I.length-o.length,baseline:_,candidate:c,delta:_===null||c===null?null:c-_,baselinePassRate:x,candidatePassRate:m,passRateDelta:x===null||m===null?null:m-x}});return{comparable:S,reasons:s,cases:I,metrics:i}}d($,"compareExperiments");export{Q as a,$ as b};