judgeval 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/dist/node/index.cjs +7 -0
  2. package/dist/{index.mjs.map → node/index.cjs.map} +11 -9
  3. package/dist/node/index.d.ts +1 -0
  4. package/dist/node/index.mjs +7 -0
  5. package/dist/{index.cjs.map → node/index.mjs.map} +11 -9
  6. package/dist/trace/BaseTracer.d.ts +5 -0
  7. package/dist/trace/BaseTracer.d.ts.map +1 -1
  8. package/dist/trace/JudgmentTracerProvider.d.ts +0 -1
  9. package/dist/trace/JudgmentTracerProvider.d.ts.map +1 -1
  10. package/dist/trace/NoOpTracer.d.ts +14 -0
  11. package/dist/trace/NoOpTracer.d.ts.map +1 -0
  12. package/dist/trace/processors/JudgmentSpanProcessor.d.ts.map +1 -1
  13. package/dist/trace/runtime.d.ts +57 -0
  14. package/dist/trace/runtime.d.ts.map +1 -0
  15. package/dist/workers/Tracer.d.ts +26 -0
  16. package/dist/workers/Tracer.d.ts.map +1 -0
  17. package/dist/workers/WorkerSpanExporter.d.ts +19 -0
  18. package/dist/workers/WorkerSpanExporter.d.ts.map +1 -0
  19. package/dist/workers/WorkerTracerProvider.d.ts +30 -0
  20. package/dist/workers/WorkerTracerProvider.d.ts.map +1 -0
  21. package/dist/workers/index.d.ts +10 -0
  22. package/dist/workers/index.d.ts.map +1 -0
  23. package/dist/workers/index.mjs +6 -0
  24. package/dist/workers/index.mjs.map +42 -0
  25. package/package.json +20 -7
  26. package/dist/index.cjs +0 -7
  27. package/dist/index.mjs +0 -7
package/dist/index.cjs DELETED
@@ -1,7 +0,0 @@
1
- var cf=Object.create;var{getPrototypeOf:rf,defineProperty:ZR,getOwnPropertyNames:K1,getOwnPropertyDescriptor:ef}=Object,u1=Object.prototype.hasOwnProperty;var hR=(R,f,q)=>{q=R!=null?cf(rf(R)):{};let $=f||!R||!R.__esModule?ZR(q,"default",{value:R,enumerable:!0}):q;for(let D of K1(R))if(!u1.call($,D))ZR($,D,{get:()=>R[D],enumerable:!0});return $},I1=new WeakMap,af=(R)=>{var f=I1.get(R),q;if(f)return f;if(f=ZR({},"__esModule",{value:!0}),R&&typeof R==="object"||typeof R==="function")K1(R).map(($)=>!u1.call(f,$)&&ZR(f,$,{get:()=>R[$],enumerable:!(q=ef(R,$))||q.enumerable}));return I1.set(R,f),f},Rq=(R,f)=>()=>(f||R((f={exports:{}}).exports,f),f.exports);var LR=(R,f)=>{for(var q in f)ZR(R,q,{get:f[q],enumerable:!0,configurable:!0,set:($)=>f[q]=()=>$})};var H=(R,f)=>()=>(R&&(f=R(R=0)),f);function NR(R,f){let q=process.env[R];if(!q)return f??null;return q}var qR,$R,DR,l1;var XR=H(()=>{qR=NR("JUDGMENT_API_KEY"),$R=NR("JUDGMENT_ORG_ID"),DR=NR("JUDGMENT_API_URL","https://api.judgmentlabs.ai"),l1=NR("JUDGMENT_LOG_LEVEL","warn")});class c{baseUrl;apiKey;organizationId;constructor(R,f,q){this.baseUrl=R,this.apiKey=f,this.organizationId=q}getBaseUrl(){return this.baseUrl}getApiKey(){return this.apiKey}getOrganizationId(){return this.organizationId}async request(R,f,q){let $=await fetch(f,{method:R,headers:{"Content-Type":"application/json",Authorization:`Bearer ${this.apiKey}`,"X-Organization-Id":this.organizationId},body:q!==void 0?JSON.stringify(q):void 0});if(!$.ok){let D=await $.text();throw Error(`HTTP ${$.status}: ${D}`)}return $.json()}async postOtelV1traces(){let R=this.baseUrl+"/otel/v1/traces";return this.request("POST",R,{})}async postOtelV1offlineTraces(){let R=this.baseUrl+"/otel/v1/offline-traces";return this.request("POST",R,{})}async postV1projectsResolve(R){let f=this.baseUrl+"/v1/projects/resolve/";return this.request("POST",f,R)}async postV1projects(R){let f=this.baseUrl+"/v1/projects";return this.request("POST",f,R)}async deleteV1projects(R){let f=this.baseUrl+`/v1/projects/${R}`;return this.request("DELETE",f,{})}async postV1projectsDatasets(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("POST",q,f)}async getV1projectsDatasets(R){let f=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("GET",f,void 0)}async getV1projectsDatasetsByDatasetIdentifierVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsDatasetsByDatasetIdentifierPage(R,f,q,$,D,P){let U=new URLSearchParams;if(q!==void 0)U.set("version",q);if($!==void 0)U.set("limit",$);if(D!==void 0)U.set("cursor_created_at",D);if(P!==void 0)U.set("cursor_example_id",P);let E=this.baseUrl+`/v1/projects/${R}/datasets/${f}/page`+(U.toString()?"?"+U.toString():"");return this.request("GET",E,void 0)}async postV1projectsDatasetsByDatasetIdentifierExamples(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/datasets/${f}/examples`;return this.request("POST",$,q)}async deleteV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("DELETE",q,{})}async getV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("GET",q,void 0)}async getV1projectsTestConfigs(R,f){let q=new URLSearchParams;if(f!==void 0)q.set("dataset_id",f);let $=this.baseUrl+`/v1/projects/${R}/test-configs`+(q.toString()?"?"+q.toString():"");return this.request("GET",$,void 0)}async postV1projectsTestConfigs(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs`;return this.request("POST",q,f)}async getV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestConfigsByTestConfigId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("PATCH",$,q)}async deleteV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("DELETE",q,{})}async getV1projectsTestRuns(R,f,q,$,D){let P=new URLSearchParams;if(f!==void 0)P.set("test_config_id",f);if(q!==void 0)P.set("dataset_id",q);if($!==void 0)P.set("status",$);if(D!==void 0)P.set("limit",D);let U=this.baseUrl+`/v1/projects/${R}/test-runs`+(P.toString()?"?"+P.toString():"");return this.request("GET",U,void 0)}async postV1projectsTestRuns(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs`;return this.request("POST",q,f)}async getV1projectsTestRunsByTestRunId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestRunsByTestRunIdStatus(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/status`;return this.request("PATCH",$,q)}async patchV1projectsTestRunsByTestRunIdSuccess(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/success`;return this.request("PATCH",$,q)}async getV1projectsTestRunsByTestRunIdItems(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/items`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdResults(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/results`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdGraph(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/graph`;return this.request("GET",q,void 0)}async getV1projectsExperimentsByRunId(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/experiments/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsEvalResults(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results`;return this.request("POST",q,f)}async postV1projectsEvalResultsExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/traces`;return this.request("POST",q,f)}async postV1projectsEvalQueueExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/examples`;return this.request("POST",q,f)}async postV1projectsEvalQueueTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/traces`;return this.request("POST",q,f)}async getV1projectsPromptsByName(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("commit_id",q);if($!==void 0)D.set("tag",$);let P=this.baseUrl+`/v1/projects/${R}/prompts/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsPrompts(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts`;return this.request("POST",q,f)}async postV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("POST",$,q)}async deleteV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("DELETE",$,q)}async getV1projectsPromptsByNameVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsScorers(R,f,q){let $=new URLSearchParams;if(f!==void 0)$.set("names",f);if(q!==void 0)$.set("is_trace",q);let D=this.baseUrl+`/v1/projects/${R}/scorers`+($.toString()?"?"+$.toString():"");return this.request("GET",D,void 0)}async getV1projectsScorersByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsScorersCustom(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom`;return this.request("POST",f,{})}async postV1projectsScorersCustomBundle(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom/bundle`;return this.request("POST",f,{})}async getV1projectsScorersCustomByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/custom/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsTracesByTraceIdTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/traces/${f}/tags`;return this.request("POST",$,q)}async getV1e2eFetchTraceByProjectNameByTraceId(R,f){let q=this.baseUrl+`/v1/e2e_fetch_trace/${R}/${f}`;return this.request("GET",q,void 0)}async getV1e2eTracesPerProject(R){let f=this.baseUrl+`/v1/e2e_traces_per_project/${R}`;return this.request("GET",f,void 0)}async postV1e2eFetchSpanScore(R){let f=this.baseUrl+"/v1/e2e_fetch_span_score/";return this.request("POST",f,R)}async postV1projectsJudges(R,f){let q=this.baseUrl+`/v1/projects/${R}/judges`;return this.request("POST",q,f)}async patchV1projectsJudgesByJudgeId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/judges/${f}`;return this.request("PATCH",$,q)}}var AR=()=>{};var F;var j=H(()=>{XR();F=class F{static RESET="\x1B[0m";static RED="\x1B[31m";static YELLOW="\x1B[33m";static GRAY="\x1B[90m";static Level={DEBUG:0,INFO:1,WARNING:2,ERROR:3,CRITICAL:4};static initialized=!1;static levelSetManually=!1;static currentLevel=F.Level.WARNING;static useColor=!0;static initialize(){if(!F.initialized){let R=process.env.JUDGMENT_NO_COLOR;if(F.useColor=!R&&process.stdout.isTTY,!F.levelSetManually){let f=l1.toLowerCase();if(f){let q={debug:F.Level.DEBUG,info:F.Level.INFO,warning:F.Level.WARNING,warn:F.Level.WARNING,error:F.Level.ERROR,critical:F.Level.CRITICAL};F.currentLevel=q[f]??F.Level.WARNING}}F.initialized=!0}}static setLevel(R){F.currentLevel=R,F.levelSetManually=!0}static setUseColor(R){F.useColor=R}static log(R,f){if(F.initialize(),R<F.currentLevel)return;let q=new Date().toISOString().replace("T"," ").substring(0,19),$=Object.keys(F.Level).find((U)=>F.Level[U]===R)??"UNKNOWN",D=`${q} - judgeval - ${$} - ${f}`;if(F.useColor)D=`${R===F.Level.DEBUG||R===F.Level.INFO?F.GRAY:R===F.Level.WARNING?F.YELLOW:F.RED}${D}${F.RESET}`;(R>=F.Level.ERROR?process.stderr:process.stdout).write(D+`
2
- `)}static debug(R){F.log(F.Level.DEBUG,R)}static info(R){F.log(F.Level.INFO,R)}static warning(R){F.log(F.Level.WARNING,R)}static warn(R){F.log(F.Level.WARNING,R)}static error(R){F.log(F.Level.ERROR,R)}static critical(R){F.log(F.Level.CRITICAL,R)}}});function BR(R){return new Promise((f)=>setTimeout(f,R))}async function p1(R,f={}){let{maxRetries:q=3,backoff:$=()=>1000,onRetry:D}=f;for(let P=1;P<=q;P++)try{return await R()}catch(U){if(P===q)throw U;D?.(P,U),await BR($(P))}throw Error("retry: exhausted all attempts")}var d1=()=>{};async function PR(R,f){let q=`org:${R.getOrganizationId()}:project:${f}`,$=s1.get(q);if($)return $;let D=H1.get(q);if(D)return D;let P=(async()=>{F.info(`Resolving project ID for project: ${f}`);let U=await p1(async()=>{let O=(await R.postV1projectsResolve({project_name:f})).project_id;if(!O)throw Error(`Project ID not found for project: ${f}`);return O},{maxRetries:3,backoff:(E)=>E*1000,onRetry:(E,O)=>{F.warning(`Failed to resolve project ID for '${f}' (attempt ${E}): ${String(O)}`)}});return F.info(`Resolved project ID: ${U}`),s1.set(q,U),U})();H1.set(q,P);try{return await P}finally{H1.delete(q)}}var s1,H1;var vR=H(()=>{j();d1();s1=new Map,H1=new Map});var GR=Rq((rq,S1)=>{var KR=process||{},n1=KR.argv||[],IR=KR.env||{},fq=!(!!IR.NO_COLOR||n1.includes("--no-color"))&&(!!IR.FORCE_COLOR||n1.includes("--color")||KR.platform==="win32"||(KR.stdout||{}).isTTY&&IR.TERM!=="dumb"||!!IR.CI),qq=(R,f,q=R)=>($)=>{let D=""+$,P=D.indexOf(f,R.length);return~P?R+$q(D,f,q,P)+f:R+D+f},$q=(R,f,q,$)=>{let D="",P=0;do D+=R.substring(P,$)+q,P=$+f.length,$=R.indexOf(f,P);while(~$);return D+R.substring(P)},g1=(R=fq)=>{let f=R?qq:()=>String;return{isColorSupported:R,reset:f("\x1B[0m","\x1B[0m"),bold:f("\x1B[1m","\x1B[22m","\x1B[22m\x1B[1m"),dim:f("\x1B[2m","\x1B[22m","\x1B[22m\x1B[2m"),italic:f("\x1B[3m","\x1B[23m"),underline:f("\x1B[4m","\x1B[24m"),inverse:f("\x1B[7m","\x1B[27m"),hidden:f("\x1B[8m","\x1B[28m"),strikethrough:f("\x1B[9m","\x1B[29m"),black:f("\x1B[30m","\x1B[39m"),red:f("\x1B[31m","\x1B[39m"),green:f("\x1B[32m","\x1B[39m"),yellow:f("\x1B[33m","\x1B[39m"),blue:f("\x1B[34m","\x1B[39m"),magenta:f("\x1B[35m","\x1B[39m"),cyan:f("\x1B[36m","\x1B[39m"),white:f("\x1B[37m","\x1B[39m"),gray:f("\x1B[90m","\x1B[39m"),bgBlack:f("\x1B[40m","\x1B[49m"),bgRed:f("\x1B[41m","\x1B[49m"),bgGreen:f("\x1B[42m","\x1B[49m"),bgYellow:f("\x1B[43m","\x1B[49m"),bgBlue:f("\x1B[44m","\x1B[49m"),bgMagenta:f("\x1B[45m","\x1B[49m"),bgCyan:f("\x1B[46m","\x1B[49m"),bgWhite:f("\x1B[47m","\x1B[49m"),blackBright:f("\x1B[90m","\x1B[39m"),redBright:f("\x1B[91m","\x1B[39m"),greenBright:f("\x1B[92m","\x1B[39m"),yellowBright:f("\x1B[93m","\x1B[39m"),blueBright:f("\x1B[94m","\x1B[39m"),magentaBright:f("\x1B[95m","\x1B[39m"),cyanBright:f("\x1B[96m","\x1B[39m"),whiteBright:f("\x1B[97m","\x1B[39m"),bgBlackBright:f("\x1B[100m","\x1B[49m"),bgRedBright:f("\x1B[101m","\x1B[49m"),bgGreenBright:f("\x1B[102m","\x1B[49m"),bgYellowBright:f("\x1B[103m","\x1B[49m"),bgBlueBright:f("\x1B[104m","\x1B[49m"),bgMagentaBright:f("\x1B[105m","\x1B[49m"),bgCyanBright:f("\x1B[106m","\x1B[49m"),bgWhiteBright:f("\x1B[107m","\x1B[49m")}};S1.exports=g1();S1.exports.createColors=g1});var m;var JR=H(()=>{m=class m{exampleId;createdAt;name;_properties;constructor(R,f,q,$){this.exampleId=R,this.createdAt=f,this.name=q,this._properties=$}static create(R={}){return new m(crypto.randomUUID(),new Date().toISOString(),null,{...R})}static META_KEYS=new Set(["example_id","created_at","name","trace_id","span_id","offline_trace_id","agent_offline_trace_id"]);static from(R){let f={};for(let q of Object.keys(R))if(!m.META_KEYS.has(q))f[q]=R[q];return new m(R.example_id??"",R.created_at??"",R.name??null,f)}get(R){return this._properties[R]}has(R){return R in this._properties}get properties(){return{...this._properties}}toJSON(){let R={example_id:this.exampleId,created_at:this.createdAt,name:this.name};for(let[f,q]of Object.entries(this._properties))R[f]=q;return R}}});function Eq(){let R=new WeakSet;return function(f,q){if(typeof q==="bigint")return q.toString();if(typeof q==="object"&&q!==null){if(R.has(q))return"[Circular]";R.add(q)}return q}}function w(R){try{let f=JSON.stringify(R);if(typeof f==="string")return f;return String(f)}catch{try{let f=JSON.stringify(R,Eq());return typeof f==="string"?f:String(R)}catch(f){return F.error(`safeStringify failed: ${f}`),String(R)}}}function e(R,f){if(typeof R==="string"||typeof R==="number"||typeof R==="boolean")return R;return f(R)}var I=H(()=>{j()});var t1="1.1.0";var o1=()=>{};var gR;var Z1=H(()=>{o1();gR=t1});var FR=()=>{};function S(R,f,q){try{return f()}catch($){let D=$ instanceof Error&&$.stack?`
3
- ${$.stack}`:"";return F.error(`[Caught] An exception was raised in ${R}: ${String($)}${D}`),q}}var y=H(()=>{j()});var c1=H(()=>{y()});function K(R,f={}){let{pre:q,post:$,error:D,finally:P}=f;return async function(...E){let O=q?S("immutableWrapAsync.pre",()=>q(...E)):void 0,Q;try{let Y=await R.apply(this,E);if($)Q=S("immutableWrapAsync.post",()=>$(O,Y,E));return Y}catch(Y){if(D)Q=S("immutableWrapAsync.error",()=>D(O,Y,E));throw Y}finally{if(P)S("immutableWrapAsync.finally",()=>{P(Q)})}}}var r1=H(()=>{y()});var e1=H(()=>{y()});function h1(R,f={}){let{pre:q,yield:$,post:D,error:P,finally:U}=f;return async function*(...O){let Q=q?S("immutableWrapAsyncIterator.pre",()=>q(...O)):void 0,Y;try{for await(let W of R(...O)){if($)S("immutableWrapAsyncIterator.yield",()=>{$(Q,W)});yield W}if(D)Y=S("immutableWrapAsyncIterator.post",()=>D(Q))}catch(W){if(P)Y=S("immutableWrapAsyncIterator.error",()=>P(Q,W));throw W}finally{if(U)S("immutableWrapAsyncIterator.finally",()=>{U(Y)})}}}var X1=H(()=>{y()});function a(R,f){let q=R[Symbol.asyncIterator].bind(R),$=h1(()=>({[Symbol.asyncIterator]:q}),{yield:(D,P)=>{f.onYield(P)},post:()=>{f.onDone()},error:(D,P)=>{f.onError(P)},finally:()=>{f.onFinally()}});R[Symbol.asyncIterator]=()=>$()}var a1=H(()=>{X1()});var bR=H(()=>{c1();r1();e1();X1();a1()});function wR(R,f){S("recordChatUsage",()=>{let q=f.prompt_tokens_details?.cached_tokens??0,$=f.prompt_tokens+f.completion_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.prompt_tokens-q:f.prompt_tokens,output_tokens:f.completion_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}var G1=H(()=>{FR();p();y();I()});function Rf(R){R.chat.completions.parse=K(R.chat.completions.parse.bind(R.chat.completions),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),q},post:(f,q)=>{if(!f)return;if(V.setOutput(w(q),f),q.usage)wR(f,q.usage);return V.recordLLMMetadata({model:q.model},f),f},error:(f,q)=>{if(f)V.setError(q,f);return f},finally:(f)=>{f?.end()}})}var ff=H(()=>{p();I();bR();G1()});function qf(R){R.chat.completions.create=K(R.chat.completions.create.bind(R.chat.completions),{pre:(f)=>{if(f.stream)f.stream_options??={include_usage:!0};let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,E="";return a(U,{onYield(O){if(typeof O.choices[0]?.delta.content==="string")E+=O.choices[0].delta.content;if(O.usage)wR(D,O.usage)},onDone(){V.setOutput(E,D)},onError(O){V.setError(O,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)wR(D,P.usage);return V.recordLLMMetadata({model:P.model},D),f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var $f=H(()=>{p();I();bR();G1()});function Df(R,f){S("images.recordUsage",()=>{let q="input_tokens_details"in f?f.input_tokens_details:void 0,$=q?.image_tokens??0;if(V.recordLLMMetadata({non_cached_input_tokens:q?.text_tokens??0,output_tokens:f.output_tokens||void 0},R),$)V.setAttribute("judgment.usage.non_cached_input_image_tokens",$,R);if(f.output_tokens)V.setAttribute("judgment.usage.output_image_tokens",f.output_tokens,R);V.setAttribute("judgment.usage.metadata",w(f),R)})}function Pf(R){R.images.generate=K(R.images.generate.bind(R.images),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,E;return a(U,{onYield(O){if(Fq.has(O.type))E=O,Df(D,E.usage)},onDone(){V.setOutput(w(E??{}),D)},onError(O){V.setError(O,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)Df(D,P.usage);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Fq;var Uf=H(()=>{FR();p();y();I();bR();Fq=new Set(["image_generation.completed","image_edit.completed"])});function Of(R,f){S("responses.recordUsage",()=>{let q=f.input_tokens_details.cached_tokens,$=f.input_tokens+f.output_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.input_tokens-q:f.input_tokens,output_tokens:f.output_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}function Ef(R){R.responses.create=K(R.responses.create.bind(R.responses),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,E="";return a(U,{onYield(O){if(O.type==="response.output_text.delta")E+=O.delta;if(O.type==="response.completed"){let Q=O.response;if(Q.usage)Of(D,Q.usage);V.recordLLMMetadata({model:Q.model},D)}},onDone(){V.setOutput(E,D)},onError(O){V.setError(O,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)Of(D,P.usage);if(typeof P.model==="string")V.recordLLMMetadata({model:P.model},D);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Vf=H(()=>{FR();p();y();I();bR()});function oR(R){return S("wrapOpenAI",()=>{qf(R),Rf(R),Ef(R),Pf(R)}),R}var Ff=H(()=>{y();ff();$f();Uf();Vf()});function tR(R){return oR(R)}var J1=H(()=>{Ff()});function Cq(R){return Function.prototype.toString.call(R)}function Zq(R){let f=Cq(R).replace(Sq,"");return f.match(Qq)||f.match(Yq)}function Qf(R){let f=Zq(R);if(!f||!f[1])return[];return f[1].split(Wq).map((q)=>{return q.replace(Hq,(D,P,U)=>U).trim()}).filter((q)=>q.length>0)}var Qq,Yq,Wq,Hq,Sq;var Yf=H(()=>{Qq=/^([^(]+?)=>/,Yq=/^[^(]*\(\s*([^)]*)\)/m,Wq=/,/,Hq=/^\s*(_?)(\S+?)\1\s*$/,Sq=/((\/\/.*$)|(\/\*[\s\S]*?\*\/))/gm});function w1(){return Hf.getStore()===!0}function Sf(R){if(b1=R,Wf)return;let f=N.context;f.active=()=>{if(!w1())return hq();let q=cR.getStore();if(q)return q;return b1?b1():N.ROOT_CONTEXT},f.with=(q,$,D,...P)=>{if(!w1())return Xq(q,$,D,...P);return cR.run(q,()=>$.apply(D,P))},f.bind=(q,$)=>{if(!w1())return Gq(q,$);if(typeof $!=="function")return $;let D=$;return(...P)=>cR.run(q,()=>D(...P))},Wf=!0}function M1(R,f){return Hf.run(!0,()=>cR.run(R,f))}var N,_1,Wf=!1,b1=null,Hf,cR,hq,Xq,Gq;var Cf=H(()=>{N=require("@opentelemetry/api"),_1=require("async_hooks"),Hf=new _1.AsyncLocalStorage,cR=new _1.AsyncLocalStorage,hq=N.context.active.bind(N.context),Xq=N.context.with.bind(N.context),Gq=N.context.bind.bind(N.context)});class Xf{_provider;constructor(R){this._provider=R}startSpan(R,f,q){let $=q??this._provider.getCurrentContext();return this._provider._getDelegateTracer().startSpan(R,f,$)}startActiveSpan(R,...f){let q={},$=this._provider.getCurrentContext(),D;if(f.length===1)D=f[0];else if(f.length===2)q=f[0],D=f[1];else q=f[0],$=f[1],D=f[2];let P=this.startSpan(R,q,$);return this._provider.useSpan(P,!1,!1,!1,()=>D(P))}}class Gf{startSpan(){return x.trace.wrapSpanContext(x.INVALID_SPAN_CONTEXT)}startActiveSpan(R,...f){return(f.length===1?f[0]:f.length===2?f[1]:f[2])(this.startSpan())}}class G{static _instance=null;_activeTracer=null;_instrumentations=[];_noOpTracer;_proxyTracer;_tracers=new Set;constructor(){this._noOpTracer=new Gf,this._proxyTracer=new Xf(this),Sf(()=>this.getCurrentContext())}static getInstance(){return G._instance??=new G,G._instance}static installAsGlobalTracerProvider(){let R=G.getInstance();return x.trace.setGlobalTracerProvider(R)}register(R){this._tracers.add(R)}deregister(R){this._tracers.delete(R)}setActive(R){let f=this.getCurrentSpan();if(f?.isRecording()){if(x.trace.getSpan(this.getCurrentContext())===f)return F.error("Cannot set_active() while a root span is active. Keeping existing tracer provider."),!1}return this.register(R),this._activeTracer=R,!0}getActiveTracer(){return this._activeTracer}getCurrentContext(){return rR.getStore()??x.ROOT_CONTEXT}setSpan(R,f){return x.trace.setSpan(R,f)}wrapSpanContext(R){return x.trace.wrapSpanContext(R)}getCurrentSpan(){let R=this.getCurrentContext();return x.trace.getSpan(R)}hasActiveRootSpan(){if(!this.getCurrentSpan()?.isRecording())return!1;return!0}_getDelegateTracer(){let R=this._activeTracer;if(!R)return F.debug("No active tracer, returning NoOpTracer"),this._noOpTracer;return R._tracerProvider.getTracer(Jq)}getTracer(R,f,q){return this._proxyTracer}addInstrumentation(R){try{Zf.registerInstrumentations({tracerProvider:this,instrumentations:[R]}),this._instrumentations.push(R)}catch(f){F.error(`Failed to add instrumentation: ${String(f)}`)}}useSpan(R,f,q,$,D){let P=this.getCurrentContext(),U=x.trace.setSpan(P,R);return rR.run(U,()=>M1(U,()=>{try{let E=D();if(E instanceof Promise)return E.catch((O)=>{if(R.isRecording()){if(q)R.recordException(O);if($){let Q=O;R.setStatus({code:x.SpanStatusCode.ERROR,message:`${Q.name}: ${Q.message}`})}}throw O}).finally(()=>{if(f)R.end()});if(f)R.end();return E}catch(E){if(R.isRecording()){if(q)R.recordException(E);if($){let O=E;R.setStatus({code:x.SpanStatusCode.ERROR,message:`${O.name}: ${O.message}`})}}if(f)R.end();throw E}}))}attachContext(R){rR.enterWith(R)}withContext(R,f){return rR.run(R,()=>M1(R,f))}async forceFlush(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.forceFlush()));for(let f of R)if(f.status==="rejected")F.error(`forceFlush failed: ${String(f.reason)}`)}async shutdown(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.shutdown()));for(let f of R)if(f.status==="rejected")F.error(`shutdown failed: ${String(f.reason)}`);this._activeTracer=null,this._tracers.clear()}}var x,Zf,hf,Jq="judgeval",rR;var u=H(()=>{j();Cf();x=require("@opentelemetry/api"),Zf=require("@opentelemetry/instrumentation"),hf=require("async_hooks"),rR=new hf.AsyncLocalStorage});var Jf="=",eR=";",_R=",",aR="baggage",bf=180,wf=4096,_f=8192;function Tf(R){return R.reduce((f,q)=>{let $=`${f}${f!==""?_R:""}${q}`;return $.length>_f?f:$},"")}function zf(R){return R.getAllEntries().map(([f,q])=>{let $=`${encodeURIComponent(f)}=${encodeURIComponent(q.value)}`;if(q.metadata!==void 0)$+=eR+q.metadata.toString();return $})}function jf(R){let f=R.split(eR);if(f.length<=0)return;let q=f.shift();if(!q)return;let $=q.indexOf(Jf);if($<=0)return;let D=decodeURIComponent(q.substring(0,$).trim()),P=decodeURIComponent(q.substring($+1).trim()),U;if(f.length>0)U=Mf.baggageEntryMetadataFromString(f.join(eR));return{key:D,value:P,metadata:U}}var Mf;var xf=H(()=>{Mf=require("@opentelemetry/api")});class d{inject(R,f,q){let $=s(R);if(!$||mf.isTracingSuppressed(R))return;let D=zf($).filter((U)=>U.length<=wf).slice(0,bf),P=Tf(D);if(P.length>0)q.set(f,aR,P)}extract(R,f,q){let $=q.get(f,aR),D=Array.isArray($)?$.join(_R):$;if(!D)return R;let P={};if(D.length===0)return R;if(D.split(_R).forEach((E)=>{let O=jf(E);if(O){let Q={value:O.value};if(O.metadata)Q.metadata=O.metadata;P[O.key]=Q}}),Object.entries(P).length===0)return R;return TR(R,MR(P))}fields(){return[aR]}}var mf;var R1=H(()=>{zR();xf();mf=require("@opentelemetry/core")});var f1={};LR(f1,{setBaggage:()=>TR,getBaggage:()=>s,getActiveBaggage:()=>bq,deleteBaggage:()=>wq,createBaggage:()=>MR,baggageEntryMetadataFromString:()=>kf.baggageEntryMetadataFromString,JudgmentBaggagePropagator:()=>d});function s(R){return R.getValue(T1)}function bq(){return s(G.getInstance().getCurrentContext())}function TR(R,f){return R.setValue(T1,f)}function wq(R){return R.deleteValue(T1)}var jR,kf,MR,T1;var zR=H(()=>{u();R1();jR=require("@opentelemetry/api"),kf=require("@opentelemetry/api"),MR=jR.propagation.createBaggage.bind(jR.propagation),T1=jR.createContextKey("baggage")});var D1={};LR(D1,{setGlobalTextmap:()=>_q,inject:()=>Mq,getGlobalTextmap:()=>z1,extract:()=>j1});function z1(){return yf}function _q(R){yf=R}function Lf(R){if(R!==void 0)return R;return G.getInstance().getCurrentContext()}function Mq(R,f,q=q1.defaultTextMapSetter){S("propagation.inject",()=>{z1().inject(Lf(f),R,q)})}function j1(R,f,q=q1.defaultTextMapGetter){let $=Lf(f);return S("propagation.extract",()=>z1().extract($,R,q),$)}var q1,$1,yf;var x1=H(()=>{y();R1();u();q1=require("@opentelemetry/api"),$1=require("@opentelemetry/core"),yf=new $1.CompositePropagator({propagators:[new $1.W3CTraceContextPropagator,new d]})});class V{projectName;projectId;apiKey;organizationId;apiUrl;environment;serializer;_tracerProvider;_client;_enableMonitoring;supportsLiveInstrumentation=!0;constructor(R,f,q,$,D,P,U,E,O,Q){this.projectName=R,this.projectId=f,this.apiKey=q,this.organizationId=$,this.apiUrl=D,this.environment=P,this.serializer=U,this._tracerProvider=E,this._client=O,this._enableMonitoring=Q}setActive(){return G.getInstance().setActive(this)}static _getProxyProvider(){return G.getInstance()}static _getSerializer(){return V._getProxyProvider().getActiveTracer()?.serializer??w}static _getCurrentTraceAndSpanId(){let f=V._getProxyProvider().getCurrentSpan();if(!f?.isRecording())return null;let q=f.spanContext();if(!q.traceId||!(q.traceFlags&1))return null;return[q.traceId,q.spanId]}static _emitPartial(){S("BaseTracer._emitPartial",()=>{let R=V._getProxyProvider().getActiveTracer();if(!R||!R.supportsLiveInstrumentation)return;R.getSpanProcessor().emitPartial()})}static getCurrentSpan(){return V._getProxyProvider().getCurrentSpan()}static async forceFlush(){await V._getProxyProvider().forceFlush()}static async shutdown(){await V._getProxyProvider().shutdown()}static registerOTELInstrumentation(R){S("BaseTracer.registerOTELInstrumentation",()=>{V._getProxyProvider().addInstrumentation(R)})}static wrap(R){return tR(R)}static getOTELTracer(){return V._getProxyProvider().getTracer(Nf)}static startSpan(R,f){let q=V.getOTELTracer().startSpan(R,{attributes:f});return V._emitPartial(),q}static startActiveSpan(R,f){let{name:q,attributes:$}=R;return V.getOTELTracer().startActiveSpan(q,{attributes:$},(D)=>{V._emitPartial();try{let P=f(D);if(P instanceof Promise)return P.finally(()=>{D.end()});return D.end(),P}catch(P){throw D.end(),P}})}static span(R,f){return V.startActiveSpan({name:R},(q)=>{try{let $=f(q);if($ instanceof Promise)return $.catch((D)=>{throw q.setStatus({code:l.SpanStatusCode.ERROR,message:String(D)}),q.recordException(D),D});return $}catch($){throw q.setStatus({code:l.SpanStatusCode.ERROR,message:String($)}),q.recordException($),$}})}static with(R,f){return V.span(R,f)}static continueTrace(R,f){let q=V._getProxyProvider(),$=j1(R);return q.withContext($,()=>f($))}static observe(R,f){let q;if(typeof R==="function")q=R;else f=R;let{spanType:$="span",spanName:D,recordInput:P=!0,recordOutput:U=!0,fork:E=!1}=f??{},O=V._getProxyProvider(),Q=(Y)=>{let W=D??Y.name;return function(...C){let J=O.getTracer(Nf);if(E&&O.getActiveTracer()!==null&&O.getCurrentSpan()?.isRecording()===!0){let h=V._getSerializer(),Z=J.startSpan(W),T=Z.spanContext();if($)Z.setAttribute("judgment.span_kind",$);let kR={"judgment.link.source_trace_id":T.traceId,"judgment.link.source_span_id":T.spanId};if($)kR["judgment.span_kind"]=$;let yR=O.setSpan(O.getCurrentContext(),O.wrapSpanContext(l.INVALID_SPAN_CONTEXT)),v=J.startSpan(W,{attributes:kR},yR),o=v.spanContext();Z.setAttribute("judgment.link.target_trace_id",o.traceId),Z.setAttribute("judgment.link.target_span_id",o.spanId);let fR=()=>{v.end(),Z.end()},B1=(z)=>{for(let L of[v,Z])L.recordException(z),L.setStatus({code:l.SpanStatusCode.ERROR,message:String(z)})},v1=(z)=>{let L=e(z,h);v.setAttribute("judgment.output",L),Z.setAttribute("judgment.output",L)};if(P){let z=e(Af(Y,C),h);v.setAttribute("judgment.input",z),Z.setAttribute("judgment.input",z)}return V._emitPartial(),O.useSpan(v,!1,!1,!1,()=>{try{let z=Y.call(this,...C);if(z instanceof Promise)return z.then((L)=>{if(U)v1(L);return L}).catch((L)=>{throw B1(L),L}).finally(fR);if(U)v1(z);return fR(),z}catch(z){throw B1(z),fR(),z}})}return J.startActiveSpan(W,(h)=>{if($)h.setAttribute("judgment.span_kind",$);try{if(P)h.setAttribute("judgment.input",e(Af(Y,C),V._getSerializer()));V._emitPartial();let Z=Y.call(this,...C);if(Z instanceof Promise)return Z.then((T)=>{if(U)h.setAttribute("judgment.output",e(T,V._getSerializer()));return T}).catch((T)=>{throw h.recordException(T),h.setStatus({code:l.SpanStatusCode.ERROR,message:String(T)}),T}).finally(()=>{h.end()});if(U)h.setAttribute("judgment.output",e(Z,V._getSerializer()));return h.end(),Z}catch(Z){throw h.recordException(Z),h.setStatus({code:l.SpanStatusCode.ERROR,message:String(Z)}),h.end(),Z}})}};if(!q)return Q;return Q(q)}static _resolveSpan(R){if(R)return R;return V._getProxyProvider().getCurrentSpan()}static setSpanKind(R,f){S("BaseTracer.setSpanKind",()=>{if(!R)return;let q=V._resolveSpan(f);if(q?.isRecording())q.setAttribute("judgment.span_kind",R)})}static setLLMSpan(){V.setSpanKind("llm")}static setToolSpan(){V.setSpanKind("tool")}static setGeneralSpan(){V.setSpanKind("span")}static setAttribute(R,f,q){S("BaseTracer.setAttribute",()=>{let $=V._resolveSpan(q);if(!$?.isRecording())return;if(!R||f==null)return;$.setAttribute(R,e(f,V._getSerializer()))})}static setAttributes(R,f){for(let[q,$]of Object.entries(R))if(f)V.setAttribute(q,$,f);else V.setAttribute(q,$)}static setInput(R,f){if(f)V.setAttribute("judgment.input",R,f);else V.setAttribute("judgment.input",R)}static setOutput(R,f){if(f)V.setAttribute("judgment.output",R,f);else V.setAttribute("judgment.output",R)}static setError(R,f){S("BaseTracer.setError",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;q.recordException(R),q.setStatus({code:l.SpanStatusCode.ERROR,message:String(R)})})}static recordLLMMetadata(R,f){S("BaseTracer.recordLLMMetadata",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;if(typeof R.model==="string")q.setAttribute("judgment.llm.model",R.model);if(typeof R.provider==="string")q.setAttribute("judgment.llm.provider",R.provider);if(typeof R.non_cached_input_tokens==="number")q.setAttribute("judgment.usage.non_cached_input_tokens",R.non_cached_input_tokens);if(typeof R.output_tokens==="number")q.setAttribute("judgment.usage.output_tokens",R.output_tokens);if(typeof R.cache_read_input_tokens==="number")q.setAttribute("judgment.usage.cache_read_input_tokens",R.cache_read_input_tokens);if(typeof R.cache_creation_input_tokens==="number")q.setAttribute("judgment.usage.cache_creation_input_tokens",R.cache_creation_input_tokens);if(typeof R.total_cost_usd==="number")q.setAttribute("judgment.usage.total_cost_usd",R.total_cost_usd)})}static _setPropagatingBaggageKey(R,f){S("BaseTracer._setPropagatingBaggageKey",()=>{let q=V._getProxyProvider(),$=q.getCurrentSpan();if(!$?.isRecording())return;$.setAttribute(R,f);let D=q.getCurrentContext(),P=(s(D)??MR()).setEntry(R,{value:f});q.attachContext(TR(D,P))})}static setCustomerId(R){V._setPropagatingBaggageKey("judgment.customer_id",R)}static setCustomerUserId(R){V._setPropagatingBaggageKey("judgment.customer_user_id",R)}static setSessionId(R){V._setPropagatingBaggageKey("judgment.session_id",R)}static tag(R){S("BaseTracer.tag",()=>{if(!R||Array.isArray(R)&&R.length===0)return;let q=V._getProxyProvider().getActiveTracer();if(!q?.projectId||!q._client)return;if(!q.supportsLiveInstrumentation)return;let $=V._getCurrentTraceAndSpanId();if(!$)return;let[D]=$,P=Array.isArray(R)?R:[R];q._client.postV1projectsTracesByTraceIdTags(q.projectId,D,{tags:P}).catch((U)=>{F.error(`tag failed: ${String(U)}`)})})}static asyncEvaluate(R,f){S("BaseTracer.asyncEvaluate",()=>{let{judge:q,example:$}=R,P=V._getProxyProvider().getActiveTracer();if(!P?.projectId)return;if(!P.supportsLiveInstrumentation)return;let U=V._resolveSpan(f);if(!U?.isRecording())return;let E=P.getSpanProcessor(),O=U.spanContext(),Q=E.stateIncr(O,"pending_evals_count"),Y={project_id:P.projectId,eval_name:`async_evaluate_${q}_${Q}`,judges:[{name:q}],examples:[{...$,example_id:Bf.randomUUID(),created_at:new Date().toISOString(),trace_id:O.traceId,span_id:O.spanId}],is_offline:!1,is_behavior:!1},W=E.stateAppend(O,"pending_evals",Y);U.setAttribute("judgment.pending_trace_eval",JSON.stringify(W))})}}function Af(R,f){try{let q=Qf(R).map((D)=>D.replace(/^\.\.\./,"").split("=")[0].trim()).filter((D)=>D.length>0),$={};return q.forEach((D,P)=>{if(P<f.length)$[D]=f[P]}),$}catch{return{}}}var l,Bf,Nf="judgeval";var p=H(()=>{FR();J1();Yf();y();j();I();zR();u();x1();l=require("@opentelemetry/api"),Bf=require("crypto")});class A{_delegate;constructor(R,f,q,$){if(!R){this._delegate=null;return}this._delegate=new vf.OTLPTraceExporter({url:R,headers:{Authorization:`Bearer ${f}`,"X-Organization-Id":q,"X-Project-Id":$}})}export(R,f){F.info(`Exported ${R.length} spans`),this._delegate?.export(R,f)}shutdown(){return this._delegate?.shutdown()??Promise.resolve()}forceFlush(){return this._delegate?.forceFlush()??Promise.resolve()}}var vf;var xR=H(()=>{j();vf=require("@opentelemetry/exporter-trace-otlp-http")});var If,n;var P1=H(()=>{xR();If=require("@opentelemetry/core");n=class n extends A{constructor(){super("","","","")}export(R,f){f({code:If.ExportResultCode.SUCCESS})}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}}});class QR{_keyPredicate;constructor(R=U1){this._keyPredicate=R}onStart(R,f){let q=s(f)?.getAllEntries()??[];for(let[$,D]of q)if(this._keyPredicate($))R.setAttribute($,D.value)}onEnd(R){}forceFlush(){return Promise.resolve()}shutdown(){return Promise.resolve()}}var U1=()=>!0;var m1=H(()=>{zR()});function YR(R){return`${R.traceId}:${R.spanId}`}function zq(R){return R[0]===0&&R[1]===0}var Kf,g;var mR=H(()=>{FR();y();u();m1();Kf=require("@opentelemetry/sdk-trace-base");g=class g extends Kf.BatchSpanProcessor{tracer;_state=new Map;_spanFinalizers;_baggageProcessor;constructor(R,f,q){super(f,q);this.tracer=R,this._spanFinalizers=new FinalizationRegistry(($)=>{this._cleanupSpanState($)}),this._baggageProcessor=new QR}_cleanupSpanState(R){this._state.delete(R)}_registerSpan(R){let f=R.spanContext();if(!f.traceId||!f.spanId)return;let q=YR(f);this._spanFinalizers.register(R,q)}stateSet(R,f,q){let $=YR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);D.set(f,q)}stateGet(R,f,q){let $=YR(R),D=this._state.get($);if(!D?.has(f))return q;return D.get(f)}stateIncr(R,f){let q=YR(R),$=this._state.get(q);if(!$)$=new Map,this._state.set(q,$);let D=$.get(f),P=typeof D==="number"?D:0;return $.set(f,P+1),P}stateAppend(R,f,q){let $=YR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);let P=D.get(f),U=Array.isArray(P)?[...P,q]:[q];return D.set(f,U),U}_emitSpan(R,f=!1){let q=R.spanContext();if(!q.traceId)return;let $=this.stateIncr(q,"judgment.update_id"),D={...R.attributes,"judgment.update_id":$};if(f)delete D["judgment.pending_trace_eval"];let P=Object.create(R);Object.defineProperty(P,"attributes",{value:D,writable:!1});let U=zq(R.endTime)?R.startTime:R.endTime;Object.defineProperty(P,"endTime",{value:U,writable:!1}),super.onEnd(P)}emitPartial(){S("JudgmentSpanProcessor.emitPartial",()=>{let f=G.getInstance().getCurrentSpan();if(!f?.isRecording())return;let q=f.spanContext();if(!q.traceId)return;if(this.stateGet(q,"disable_partial_emit",!1))return;this._emitSpan(f,!0)})}onStart(R,f){S("JudgmentSpanProcessor.onStart",()=>{this._baggageProcessor.onStart(R,f),this._registerSpan(R)})}onEnd(R){S("JudgmentSpanProcessor.onEnd",()=>{let f=R.spanContext();if(!f.traceId){super.onEnd(R);return}let q=YR(f);try{if(!this.stateGet(f,"cancelled",!1))this._emitSpan(R)}finally{this._cleanupSpanState(q)}})}}});var WR;var k1=H(()=>{P1();mR();WR=class WR extends g{constructor(){super(null,new n)}onStart(R,f){}onEnd(R){}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}emitPartial(){}stateSet(R,f,q){}stateGet(R,f,q){return q}stateIncr(R,f){return 0}stateAppend(R,f,q){return[q]}}});var O1,y1,B;var E1=H(()=>{XR();AR();j();vR();I();Z1();p();u();xR();P1();mR();k1();O1=require("@opentelemetry/resources"),y1=require("@opentelemetry/sdk-trace-node");B=class B extends V{_spanExporter=null;_spanProcessor=null;constructor(R,f,q,$,D,P,U,E,O,Q){super(R,f,q,$,D,P,U,E,O,Q)}static async init(R={}){let f=R.apiKey??qR,q=R.organizationId??$R,$=R.apiUrl??DR,D=R.projectName??null,P=R.serializer??w,U=!0;if(!D)F.warning("project_name not provided. Tracer will not export spans."),U=!1;if(!f)F.warning("api_key not provided. Tracer will not export spans."),U=!1;if(!q)F.warning("organization_id not provided. Tracer will not export spans."),U=!1;if(!$)F.warning("api_url not provided. Tracer will not export spans."),U=!1;let E=null,O=null;if(U&&D&&f&&q&&$){if(E=new c($,f,q),O=await PR(E,D).catch(()=>null),!O)F.warning(`Project '${D}' not found. Tracer will not export spans.`),U=!1}let Q={"service.name":D??"unknown","telemetry.sdk.name":"judgeval","telemetry.sdk.version":gR};if(R.environment)Q["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(Q,R.resourceAttributes);let Y=O1.defaultResource().merge(O1.resourceFromAttributes(Q)),W=new y1.NodeTracerProvider({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits}),C=new B(D,O,f,q,$,R.environment??null,P,W,E,U);if(U){let b=new y1.NodeTracerProvider({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[C.getSpanProcessor(),...R.spanProcessors??[]]});C._tracerProvider=b}if(G.getInstance().register(C),R.setActive??!0)C.setActive();return C}getSpanExporter(){if(this._spanExporter)return this._spanExporter;if(!this._enableMonitoring||!this.projectId||!this.apiKey||!this.organizationId||!this.apiUrl)this._spanExporter=new n;else{let R=this.apiUrl.endsWith("/")?this.apiUrl+"otel/v1/traces":this.apiUrl+"/otel/v1/traces";this._spanExporter=new A(R,this.apiKey,this.organizationId,this.projectId)}return this._spanExporter}getSpanProcessor(){if(this._spanProcessor)return this._spanProcessor;if(!this._enableMonitoring)this._spanProcessor=new WR;else this._spanProcessor=new g(this,this.getSpanExporter());return this._spanProcessor}}});var HR;var L1=H(()=>{JR();mR();HR=class HR extends g{_dataset;_exampleFields;_seenTraceIds=new Set;constructor(R,f,q){super(R,f);this._dataset=q.dataset,this._exampleFields={...q.exampleFields??{}}}_maybeCreateExample(R){if(R.parentSpanContext)return;let f=R.spanContext();if(!f?.traceId)return;if(this._seenTraceIds.has(f.traceId))return;this._seenTraceIds.add(f.traceId);let q=m.create({...this._exampleFields,offline_trace_id:f.traceId});this._dataset.push(q)}onEnd(R){try{this._maybeCreateExample(R)}finally{super.onEnd(R)}}}});var lf={};LR(lf,{OfflineTracer:()=>i});var V1,N1,uf="otel/v1/offline-traces",i;var F1=H(()=>{XR();AR();vR();I();Z1();u();E1();xR();L1();V1=require("@opentelemetry/resources"),N1=require("@opentelemetry/sdk-trace-node");i=class i extends B{supportsLiveInstrumentation=!1;_offlineApiUrl;_offlineApiKey;_offlineOrganizationId;_offlineProjectId;_dataset;_exampleFields;_offlineSpanExporter=null;_offlineSpanProcessor=null;constructor(R){super(R.projectName,R.projectId,R.apiKey,R.organizationId,R.apiUrl,R.environment,R.serializer,R.tracerProvider,R.client,!0);this._offlineApiUrl=R.apiUrl,this._offlineApiKey=R.apiKey,this._offlineOrganizationId=R.organizationId,this._offlineProjectId=R.projectId,this._dataset=R.dataset,this._exampleFields=R.exampleFields}static async create(R){let f=R.apiKey??qR,q=R.organizationId??$R,$=R.apiUrl??DR,D=R.projectName,P=R.serializer??w;if(!D)throw Error("projectName is required for OfflineTracer");if(!f)throw Error("apiKey is required for OfflineTracer");if(!q)throw Error("organizationId is required for OfflineTracer");if(!$)throw Error("apiUrl is required for OfflineTracer");let U=new c($,f,q),E;try{E=await PR(U,D)}catch(J){throw Error(`Project '${D}' not found; cannot start OfflineTracer: ${String(J)}`)}let O={"service.name":D,"telemetry.sdk.name":"judgeval","telemetry.sdk.version":gR,"judgment.offline":"true"};if(R.environment)O["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(O,R.resourceAttributes);let Q=V1.defaultResource().merge(V1.resourceFromAttributes(O)),Y=new i({projectName:D,projectId:E,apiKey:f,organizationId:q,apiUrl:$,environment:R.environment??null,serializer:P,tracerProvider:new N1.NodeTracerProvider({resource:Q}),client:U,dataset:R.dataset,exampleFields:{...R.exampleFields??{}}}),W=new N1.NodeTracerProvider({resource:Q,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[Y.getSpanProcessor(),...R.spanProcessors??[]]});if(Y._tracerProvider=W,G.getInstance().register(Y),R.setActive??!0)Y.setActive();return Y}getSpanExporter(){if(this._offlineSpanExporter)return this._offlineSpanExporter;let R=this._offlineApiUrl.endsWith("/")?this._offlineApiUrl+uf:this._offlineApiUrl+"/"+uf;return this._offlineSpanExporter=new A(R,this._offlineApiKey,this._offlineOrganizationId,this._offlineProjectId),this._offlineSpanExporter}getSpanProcessor(){if(this._offlineSpanProcessor)return this._offlineSpanProcessor;return this._offlineSpanProcessor=new HR(this,this.getSpanExporter(),{dataset:this._dataset,exampleFields:this._exampleFields}),this._offlineSpanProcessor}}});var Aq={};LR(Aq,{wrapOpenAI:()=>oR,wrap:()=>tR,validateDatasetSchema:()=>dR,propagation:()=>D1,inferSchemaFromExamples:()=>sR,baggage:()=>f1,Tracer:()=>B,OfflineTracer:()=>i,OfflineTestsFactory:()=>CR,OfflineTestRunner:()=>SR,OfflineJudgmentSpanProcessor:()=>HR,NoOpSpanProcessor:()=>WR,NoOpSpanExporter:()=>n,JudgmentTracerProvider:()=>G,JudgmentSpanExporter:()=>A,JudgmentBaggageSpanProcessor:()=>QR,JudgmentBaggagePropagator:()=>d,Judgeval:()=>W1,Judge:()=>UR,Example:()=>m,Evaluation:()=>ER,Dataset:()=>r,BaseTracer:()=>V,AgentJudgeFactory:()=>VR,ALLOW_ALL_BAGGAGE_KEYS:()=>U1});module.exports=af(Aq);XR();AR();vR();j();class UR{}j();var C1=hR(GR(),1);j();var X=hR(GR(),1),Dq=2000;class OR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async _poll(R,f,q,$){let D=Date.now();while(!0){let P=(Date.now()-D)/1000;if(P>$)throw Error(`Evaluation timed out after ${$}s`);let U=await this._client.getV1projectsExperimentsByRunId(R,f),E=U.results??[];if(E.length>=q){let Q=U.ui_results_url??"Failed to get UI results URL";return console.log(`${X.default.green("✓")} Evals completed and saved in ${X.default.bold(`${P.toFixed(1)}s`)}`),{results:E,url:Q}}await new Promise((Q)=>setTimeout(Q,Dq))}}_displayResults(R,f,q,$){let D=[],P=0,U=0;console.log();for(let E=0;E<f.length;E++){let O=f[E],Q=O.scorers.every((Y)=>Boolean(Y.success));if(Q)P++,console.log(`${X.default.green("✓")} Example ${E+1}: ${X.default.green("PASSED")}`);else U++,console.log(`${X.default.red("✗")} Example ${E+1}: ${X.default.red("FAILED")}`);for(let Y of O.scorers){let W=Y.score!==null?Y.score.toFixed(3):"N/A",C=Y.success?X.default.green(W):X.default.red(W);console.log(` ${X.default.dim(`${Y.name}:`)} ${C} ${X.default.dim(`(threshold: ${Y.threshold})`)}`)}D.push({success:Q,scorers:O.scorers,example:R[E]})}if(console.log(),P===D.length)console.log(`${X.default.bold(X.default.green("✓ All tests passed!"))} (${P}/${D.length})`);else console.log(`${X.default.bold(X.default.yellow("⚠ Results:"))} ${X.default.green(`${P} passed`)} | ${X.default.red(`${U} failed`)}`);if(console.log(`${X.default.dim("View full details:")} ${X.default.underline(q)}`),console.log(),$&&D.some((E)=>!E.success)){let E=[`Evaluation failed: ${U}/${D.length} examples failed`];for(let O=0;O<D.length;O++)if(!D[O].success){E.push(` Example ${O+1}:`);for(let Q of D[O].scorers)if(!Q.success){if(E.push(` ${Q.name}: ${Q.score!==null?Q.score.toFixed(3):"N/A"} (threshold: ${Q.threshold})`),Q.reason)E.push(` ${Q.reason}`)}}throw Error(E.join(`
4
- `))}return D}async run(R,f,q,$=!1,D=300){if(!this._projectId)return F.error("Project ID is not resolved. Evaluation requires a valid project."),[];let P=this._projectId,U=crypto.randomUUID(),E=new Date().toISOString();console.log(),console.log(X.default.bold(X.default.cyan("Starting Evaluation"))),console.log(`${X.default.dim("Run:")} ${q}`),console.log(`${X.default.dim("Project:")} ${this._projectName}`),console.log(`${X.default.dim("Examples:")} ${R.length} | ${X.default.dim("Scorers:")} ${f.length}`),console.log();let O=this._buildPayload(U,P,q,E,R,f),Q=await this._submit(P,U,R,f,O),{results:Y,url:W}=await this._poll(P,U,Q,D);return this._displayResults(R,Y,W,$)}}class uR extends OR{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:[],custom_scorers:[]}}async _submit(R,f,q,$,D){let P=Date.now(),U=q.flatMap((W,C)=>$.map((J)=>J.score(W).then((b)=>({exampleIdx:C,scorer:J,result:b,error:null})).catch((b)=>({exampleIdx:C,scorer:J,result:null,error:String(b)})))),E=await Promise.all(U),O=((Date.now()-P)/1000).toFixed(1);console.log(`${C1.default.green("✓")} Scoring completed in ${C1.default.bold(`${O}s`)}`);let Q=new Map;for(let W of E){let C=Q.get(W.exampleIdx);if(!C)C=[],Q.set(W.exampleIdx,C);C.push(W)}let Y=q.map((W,C)=>{return{scorers_data:(Q.get(C)??[]).map((b)=>{if(b.error!==null)return{scorer_name:b.scorer.constructor.name,value:0,reason:"",error:b.error};let h=b.result;return{scorer_name:b.scorer.constructor.name,value:h.value,reason:h.reason,...h.citations&&{citations:h.citations.map((Z)=>({span_id:Z.spanId,span_attribute:Z.spanAttribute}))}}}),data_object:W.toJSON()}});return await this._client.postV1projectsEvalResultsExamples(R,{results:Y,run:D}),q.length}}var i1=hR(GR(),1);class lR extends OR{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:P.map((U)=>({name:U})),custom_scorers:[]}}async _submit(R,f,q,$,D){return await this._client.postV1projectsEvalQueueExamples(R,D),console.log(`${i1.default.green("✓")} Evaluation submitted`),q.length}}class ER{_local;_hosted;constructor(R,f,q){this._local=new uR(R,f,q),this._hosted=new lR(R,f,q)}run(R){let{examples:f,scorers:q,evalRunName:$,assertTest:D=!1,timeoutSeconds:P=300}=R,U=q.filter((O)=>typeof O==="string"),E=q.filter((O)=>O instanceof UR);if(E.length>0&&U.length>0)return F.error("Running both local and hosted scorers is not supported. Please run your evaluation with either local or hosted scorers, but not both."),Promise.resolve([]);if(E.length===0&&U.length===0)return F.error("No valid local or hosted scorers provided. Please provide at least one local or hosted scorer."),Promise.resolve([]);if(E.length>0)return this._local.run(f,E,$,D,P);return this._hosted.run(f,U,$,D,P)}}class pR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}create(){return new ER(this._client,this._projectId,this._projectName)}}JR();j();JR();class r{name;projectId;projectName;datasetKind;examples;_client;constructor(R){this.name=R.name,this.projectId=R.projectId,this.projectName=R.projectName,this.datasetKind=R.datasetKind??"example",this.examples=R.examples??[],this._client=R.client??null}async addExamples(R,f=100){if(!this._client)return;for(let q=0;q<R.length;q+=f){let $=R.slice(q,q+f);await this._client.postV1projectsDatasetsByDatasetIdentifierExamples(this.projectId,this.name,{examples:$.map((D)=>D.toJSON())})}}async addFromJson(R,f=100){let{readFile:q}=await import("fs/promises"),$=await q(R,"utf-8"),P=JSON.parse($).map((U)=>{if(typeof U!=="object"||U===null)throw Error("Each item in the JSON array must be an object");return m.create(U)});await this.addExamples(P,f)}get length(){return this.examples.length}[Symbol.iterator](){return this.examples[Symbol.iterator]()}toString(){return`Dataset(name=${this.name}, examples=${this.examples.length})`}}var Pq="trace";function dR(R){if(typeof R!=="object"||R===null||Array.isArray(R))throw Error("Dataset schema must be a JSON object.");let f=R;if(f.type!=="object")throw Error('Dataset schema must declare top-level type "object".');let q=f.properties;if(typeof q!=="object"||q===null||Array.isArray(q))throw Error("Dataset schema must declare a 'properties' object.");let $=Object.entries(q).filter(([,D])=>typeof D==="object"&&D!==null&&D.type===Pq).map(([D])=>D);if($.length>1)throw Error(`A dataset may declare at most one trace column; found ${$.length}: ${$.join(", ")}.`)}function Uq(R){if(typeof R==="boolean")return"boolean";if(typeof R==="number")return Number.isInteger(R)?"integer":"number";if(typeof R==="string")return"string";if(Array.isArray(R))return"array";if(typeof R==="object"&&R!==null)return"object";return"string"}function sR(R){if(R.length===0)throw Error("Cannot infer a dataset schema without examples. Pass an explicit `schema` to client.datasets.create().");let f={},q=null;for(let $ of R){let D=new Set;for(let[P,U]of Object.entries($.properties)){if(U===null||U===void 0)continue;if(D.add(P),!(P in f))f[P]={type:Uq(U)}}if(q===null)q=D;else if(q.size!==D.size||![...q].every((P)=>D.has(P)))throw Error("All examples must share the same set of fields (dataset schemas require all declared properties). Pass an explicit `schema` or make all examples homogeneous.")}return{type:"object",properties:f}}class nR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async get(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsDatasetsByDatasetIdentifier(f,R),$=q.dataset_kind??"example",D=(q.examples??[]).map((P)=>m.from({...P.data??{},example_id:P.example_id,created_at:P.created_at??""}));return new r({name:R,projectId:f,projectName:this._projectName,datasetKind:$,examples:D,client:this._client})}async create(R,f={}){let q=this._expectProjectId();if(!q)return null;let{examples:$=[],overwrite:D=!1,batchSize:P=100,schema:U}=f,E;if(U!==void 0)dR(U),E=U;else{if($.length===0)throw Error("Datasets require a JSON Schema. Pass `schema` to create(), or provide `examples` to infer one.");E=sR($)}await this._client.postV1projectsDatasets(q,{name:R,examples:[],dataset_kind:"example",overwrite:D,schema:E});let O=new r({name:R,projectId:q,projectName:this._projectName,examples:$,client:this._client});if($.length>0)await O.addExamples($,P);return O}list(){let R=this._expectProjectId();if(!R)return Promise.resolve(null);return this._client.getV1projectsDatasets(R)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Dataset operations require a valid project."),null;return this._projectId}}j();class VR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async create(R){let f=this._expectProjectId();if(!f)return null;let q={name:R.name,prompt:R.prompt,model:R.model,score_type:R.scoreType};if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;return{judgeId:(await this._client.postV1projectsJudges(f,q)).judge_id,name:R.name,prompt:R.prompt,model:R.model,scoreType:R.scoreType,description:R.description??null,judgeDescription:R.judgeDescription??null,categories:R.categories??null,minScore:R.minScore??null,maxScore:R.maxScore??null,majorVersion:0,minorVersion:0}}async update(R){let f=this._expectProjectId();if(!f)return null;let q={};if(R.prompt!==void 0)q.prompt=R.prompt;if(R.model!==void 0)q.model=R.model;if(R.scoreType!==void 0)q.score_type=R.scoreType;if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;if(R.sourceMajorVersion!==void 0)q.source_major_version=R.sourceMajorVersion;if(R.sourceMinorVersion!==void 0)q.source_minor_version=R.sourceMinorVersion;if(R.targetMajorVersion!==void 0)q.target_major_version=R.targetMajorVersion;if(R.targetMinorVersion!==void 0)q.target_minor_version=R.targetMinorVersion;let $=await this._client.patchV1projectsJudgesByJudgeId(f,R.judgeId,q);return Oq($)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Agent judge operations require a valid project."),null;return this._projectId}}function Oq(R){let f=R.judge;return{judgeId:f.id,name:f.name,prompt:f.prompt??"",model:f.model??"",scoreType:f.score_type,description:f.description??null,judgeDescription:f.judge_description??null,categories:f.categories??null,minScore:f.min_score??null,maxScore:f.max_score??null,majorVersion:f.major_version??null,minorVersion:f.minor_version??null}}j();j();E1();F1();u();function Q1(R){let f=Array.isArray(R.judges)?R.judges.filter((q)=>typeof q==="object"&&q!==null):[];return{id:typeof R.id==="string"?R.id:"",name:typeof R.name==="string"?R.name:"",datasetId:typeof R.dataset_id==="string"?R.dataset_id:"",description:typeof R.description==="string"?R.description:null,createdAt:typeof R.created_at==="string"?R.created_at:null,judges:f}}function pf(R){let f=[];for(let q of R)for(let $ of q.scorers)if($.success!==null&&$.success!==void 0)f.push($.success);if(f.length===0)return null;return f.every((q)=>q)}var _=hR(GR(),1);function k(R){return typeof R==="object"&&R!==null&&!Array.isArray(R)?R:{}}function RR(R){return Array.isArray(R)?R:[]}function M(R){return typeof R==="string"?R:""}function df(R){return R?"Yes":"No"}function jq(R){let{score_type:f,bool_value:q,str_value:$,num_value:D}=R;if(f==="binary")return typeof q==="boolean"?df(q):null;if(f==="categorical")return typeof $==="string"?$:null;if(f==="numeric")return typeof D==="number"?D:null;if(typeof q==="boolean")return df(q);if(typeof $==="string")return $;if(typeof D==="number")return D;return null}function xq(R){if(typeof R==="object"&&R!==null){let f=R.text;return typeof f==="string"&&f?f:null}if(typeof R==="string"){try{let f=JSON.parse(R);if(typeof f==="object"&&f!==null){let q=f.text;if(typeof q==="string")return q||null}}catch{}return R||null}return null}function sf(R){if(!R||R.length===0)return;let f=[];for(let q of R){if(!q.name&&!q.judgeId)throw Error("judgeVersions entries require a 'name' (or 'judgeId') key");let $={};if(q.judgeId!==void 0)$.judge_id=q.judgeId;if(q.name!==void 0)$.name=q.name;if(q.tag!==void 0)$.tag=q.tag;if(q.version!==void 0)$.version=q.version;if(q.majorVersion!==void 0)$.major_version=q.majorVersion;if(q.minorVersion!==void 0)$.minor_version=q.minorVersion;f.push($)}return f}function nf(R,f,q){let $=[];for(let D of R){let P=M(D.example_id),U=k(D.data),E=[];for(let O of RR(D.scorers)){let Q=k(O),Y={judge_id:Q.judge_id,judge_major_version:Q.judge_major_version,judge_minor_version:Q.judge_minor_version},W=xq(Q.reason);if(W)Y.reason=W;E.push({name:M(Q.judge_name),value:jq(Q),scoreType:typeof Q.score_type==="string"?Q.score_type:null,error:typeof Q.error==="string"?Q.error:null,success:typeof Q.success==="boolean"?Q.success:null,metadata:Y})}if(q){let O=Boolean(q({...U},E));for(let Q of E)Q.success=O}$.push({exampleId:P,data:U,scorers:E,agentOfflineTraceId:f[P]})}return $}function gf(R){if(R.status!=="completed")throw Error(`Test run ${R.testRunId} finished with status '${R.status}'`);let f=R.results.filter((q)=>q.scorers.some(($)=>$.success===!1)).map((q)=>q.exampleId);if(f.length>0||R.passed!==!0)throw Error(`Test run ${R.testRunId} failed its pass condition for ${f.length} example(s): ${JSON.stringify(f)}`)}function of(R,f,q){console.log(),console.log(_.default.bold(_.default.cyan("Starting Offline Test"))),console.log(`${_.default.dim("Config:")} ${R}`),console.log(`${_.default.dim("Project:")} ${f}`),console.log(`${_.default.dim("Examples:")} ${q}`),console.log()}function tf(R,f){if(console.log(),R.forEach((q,$)=>{if(q.scorers.some((P)=>P.success!==null&&P.success!==void 0)){let P=q.scorers.every((U)=>U.success===!0);console.log(P?`${_.default.green("✓")} Example ${$+1}: ${_.default.green("PASSED")}`:`${_.default.red("✗")} Example ${$+1}: ${_.default.red("FAILED")}`)}else console.log(`${_.default.cyan("•")} Example ${$+1}:`);for(let P of q.scorers){if(P.error){console.log(` ${_.default.dim(`${P.name}:`)} ${_.default.red(`error: ${P.error}`)}`);continue}let U=P.value===null?"N/A":String(P.value),E=P.success===!0?_.default.green(U):P.success===!1?_.default.red(U):U;console.log(` ${_.default.dim(`${P.name}:`)} ${E}`)}}),console.log(),f)console.log(`${_.default.dim("View full details:")} ${_.default.underline(f)}`),console.log()}var mq=new Set(["completed","error","cancelled"]),kq=100,yq=200,Lq=2000,Y1=1e4;class SR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async resolveDatasetVersion(R,f){let q=await this._client.getV1projectsDatasetsByDatasetIdentifierVersions(this._projectId,R.datasetId),$=RR(q.versions).filter((D)=>typeof D==="object"&&D!==null).map((D)=>D);if(f===void 0){if($.length===0)throw Error(`Dataset of test config '${R.name}' has no versions`);return $.reduce((D,P)=>Number(P.version_number??0)>Number(D.version_number??0)?P:D)}if(typeof f==="number"){let D=$.find((P)=>Number(P.version_number??0)===f);if(D)return D}else{let D=$.find((P)=>P.version_id===f);if(D)return D}throw Error(`Dataset version ${JSON.stringify(f)} does not exist for the dataset of test config '${R.name}'`)}async fetchExamples(R,f){let q=[],$,D,P=0;for(;;){if(P>=Y1)throw Error(`fetchExamples exceeded ${Y1} pages for dataset of test config '${R.name}'`);let U=await this._client.getV1projectsDatasetsByDatasetIdentifierPage(this._projectId,R.datasetId,String(f),String(kq),$,D);P+=1;for(let Q of RR(U.entries)){let Y=k(k(Q).example),W=Y.data;if(typeof W==="string")try{W=JSON.parse(W)}catch{W={}}q.push({exampleId:M(Y.example_id),data:k(W),offlineTraceId:typeof Y.offline_trace_id==="string"?Y.offline_trace_id:null,createdAt:typeof Y.created_at==="string"?Y.created_at:null})}let E=k(U.metadata),O=k(E.nextCursor);if(!E.hasMore||Object.keys(O).length===0)break;$=M(O.created_at),D=M(O.example_id)}return q}createTestRun(R,f={}){let q={test_config_id:R.id,source:f.source??"sdk"};if(f.name)q.name=f.name;if(typeof f.datasetVersion==="number")q.dataset_version_number=f.datasetVersion;else if(typeof f.datasetVersion==="string")q.dataset_version_id=f.datasetVersion;let $=sf(f.judgeVersions);if($)q.judge_versions=$;let D=f.agentTraces??{};if(Object.keys(D).length>0)q.agent_traces=Object.entries(D).map(([P,U])=>({example_id:P,agent_offline_trace_id:U}));return this._client.postV1projectsTestRuns(this._projectId,q)}async runAgent(R,f){let q=[],$=G.getInstance().getActiveTracer();await i.create({projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl(),dataset:q,setActive:!0});let D={};try{let P=B.observe(R,{spanType:"agent"});for(let U of f){let E=q.length;try{await P(U.data)}catch(O){F.error(`Agent entrypoint raised for example ${U.exampleId}: ${String(O)}`)}for(let O of q.slice(E)){let Q=O.get("offline_trace_id");if(U.exampleId&&typeof Q==="string"){D[U.exampleId]=Q;break}}}}finally{if(await B.forceFlush(),$)$.setActive()}return D}async waitForCompletion(R,f){let q=Date.now();for(;;){if((Date.now()-q)/1000>f)throw Error(`Test run ${R} did not complete within ${f}s`);let $=await this._client.getV1projectsTestRunsByTestRunId(this._projectId,R),D=M(k($.test_run).status);if(mq.has(D))return D;await BR(Lq)}}async fetchItems(R){let f=[],q="",$,D=0;for(;;){if(D>=Y1)throw Error(`fetchItems exceeded ${Y1} pages for test run ${R}`);let P=await this._client.getV1projectsTestRunsByTestRunIdItems(this._projectId,R,String(yq),$);D+=1;for(let E of RR(P.results))f.push(k(E));if(!q)q=M(P.ui_results_url);let U=P.next_cursor;if(!P.has_more||typeof U!=="string"||!U)break;$=U}return{items:f,uiResultsUrl:q}}async reportSuccess(R,f,q,$){let D=new Map,P=new Map,U=(Y,W,C,J)=>`${Y}\x00${W}\x00${C}\x00${J}`,E=(Y,W)=>`${Y}\x00${W}`;for(let Y of RR(f.evaluation_runs)){let W=k(Y),C=W.run_id;if(typeof C!=="string"||!C)continue;D.set(U(M(W.example_id),M(W.judge_id),Number(W.judge_major_version??0),Number(W.judge_minor_version??0)),C),P.set(E(M(W.example_id),M(W.judge_name)),C)}let O=new Map;for(let Y of $)O.set(Y.exampleId,Y);let Q=[];for(let Y of q){let W=M(Y.example_id),C=O.get(W),J=C?C.scorers.map((h)=>h.success??null):[],b=RR(Y.scorers);for(let h=0;h<b.length;h+=1){let Z=k(b[h]),T=D.get(U(W,M(Z.judge_id),Number(Z.judge_major_version??0),Number(Z.judge_minor_version??0)))??P.get(E(W,M(Z.judge_name)));if(!T){F.warning(`No evaluation run ref for scorer ${M(Z.judge_name)} of example ${W}; skipping its success update`);continue}Q.push({evaluation_run_id:T,success:h<J.length?J[h]:null})}}if(Q.length===0)return;await this._client.patchV1projectsTestRunsByTestRunIdSuccess(this._projectId,R,{successes:Q})}async run(R,f={}){let{agentFunction:q,judgeVersions:$,datasetVersion:D,passConditionFn:P,assertTest:U=!1,timeoutSeconds:E=600,runName:O}=f;if(U&&!P)throw Error("assertTest=true requires a passConditionFn to decide whether each row passes.");let Q=await this.resolveDatasetVersion(R,D),Y=Number(Q.version_number??0),W=await this.fetchExamples(R,Y);of(R.name,this._projectName,W.length),F.info(`Dataset version ${Y}: ${W.length} example(s)`);let C=typeof D==="string"?D:Y,J={};if(q&&W.length>0)J=await this.runAgent(q,W);let b=await this.createTestRun(R,{datasetVersion:C,judgeVersions:$,agentTraces:J,name:O}),h=k(b.test_run),Z=M(h.id),T=M(b.ui_results_url);F.info(`Created test run ${Z} over ${W.length} examples`);let kR=await this.waitForCompletion(Z,E),{items:yR,uiResultsUrl:v}=await this.fetchItems(Z);if(v)T=v;let o=nf(yR,J,P);if(P)await this.reportSuccess(Z,b,yR,o);tf(o,T||void 0);let fR={testRunId:Z,status:kR,uiResultsUrl:T||void 0,results:o,agentOfflineTraceIds:J,passed:pf(o)};if(U)gf(fR);return fR}}var Nq=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function A1(R){return Nq.test(R)}class CR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async createConfig(R,f,q,$){let D=this._expectProjectId();if(!D)return null;let P=[];for(let O of q)if(typeof O==="string")P.push(A1(O)?{judge_id:O}:{name:O});else if(O.judgeId)P.push({judge_id:O.judgeId});else if(O.name)P.push({name:O.name});else throw Error("judges entries must be judge names or objects with a 'judgeId' or 'name'");let U={name:R,judges:P};if($!==void 0)U.description=$;if(A1(f))U.dataset_id=f;else U.dataset_name=f;let E=await this._client.postV1projectsTestConfigs(D,U);return F.info(`Created test config ${R}`),Q1(E.test_config??{})}async getConfig(R){let f=this._expectProjectId();if(!f)return null;if(A1(R)){let D=await this._client.getV1projectsTestConfigsByTestConfigId(f,R);return Q1(D.test_config??{})}let $=(await this.listConfigs()??[]).find((D)=>D.name===R);if(!$)return F.error(`Test config '${R}' not found`),null;return $}async listConfigs(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsTestConfigs(f,R);return(Array.isArray(q.test_configs)?q.test_configs:[]).map((D)=>Q1(D))}async deleteConfig(R){let f=this._expectProjectId();if(!f)return!1;return await this._client.deleteV1projectsTestConfigsByTestConfigId(f,R),F.info(`Deleted test config ${R}`),!0}async run(R,f={}){let q=this._expectProjectId();if(!q)return null;let $=typeof R==="string"?await this.getConfig(R):R;if(!$||!$.id)return null;return new SR(this._client,q,this._projectName).run($,f)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Offline-test operations require a valid project."),null;return this._projectId}}class W1{_client;_projectName;_projectId;constructor(R,f,q){this._client=R,this._projectName=f,this._projectId=q}static async create(R){let f=R.apiKey??qR,q=R.organizationId??$R,$=R.apiUrl??DR;if(!f)throw Error("API key is required");if(!q)throw Error("Organization ID is required");if(!$)throw Error("API URL is required");if(!R.projectName)throw Error("Project name is required");let D=new c($,f,q),P=null;try{P=await PR(D,R.projectName)}catch{F.warning(`Project '${R.projectName}' not found. Some operations requiring project_id will be skipped.`)}return new W1(D,R.projectName,P)}async offlineTracer(R){let{OfflineTracer:f}=await Promise.resolve().then(() => (F1(),lf));return f.create({...R,projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl()})}get datasets(){return new nR(this._client,this._projectId,this._projectName)}get evaluation(){return new pR(this._client,this._projectId,this._projectName)}get agentJudges(){return new VR(this._client,this._projectId,this._projectName)}get offlineTests(){return new CR(this._client,this._projectId,this._projectName)}}p();xR();P1();u();mR();k1();L1();m1();R1();zR();x1();E1();F1();J1();JR();
5
-
6
- //# debugId=A4B1A6E1DDC7F86A64756E2164756E21
7
- //# sourceMappingURL=index.cjs.map
package/dist/index.mjs DELETED
@@ -1,7 +0,0 @@
1
- import{createRequire as Kf}from"node:module";var Nf=Object.create;var{getPrototypeOf:Af,defineProperty:aR,getOwnPropertyNames:Bf}=Object;var vf=Object.prototype.hasOwnProperty;var VR=(R,f,q)=>{q=R!=null?Nf(Af(R)):{};let $=f||!R||!R.__esModule?aR(q,"default",{value:R,enumerable:!0}):q;for(let D of Bf(R))if(!vf.call($,D))aR($,D,{get:()=>R[D],enumerable:!0});return $};var If=(R,f)=>()=>(f||R((f={exports:{}}).exports,f),f.exports);var R1=(R,f)=>{for(var q in f)aR(R,q,{get:f[q],enumerable:!0,configurable:!0,set:($)=>f[q]=()=>$})};var H=(R,f)=>()=>(R&&(f=R(R=0)),f);var uf=Kf(import.meta.url);function kR(R,f){let q=process.env[R];if(!q)return f??null;return q}var RR,fR,qR,m1;var FR=H(()=>{RR=kR("JUDGMENT_API_KEY"),fR=kR("JUDGMENT_ORG_ID"),qR=kR("JUDGMENT_API_URL","https://api.judgmentlabs.ai"),m1=kR("JUDGMENT_LOG_LEVEL","warn")});class s{baseUrl;apiKey;organizationId;constructor(R,f,q){this.baseUrl=R,this.apiKey=f,this.organizationId=q}getBaseUrl(){return this.baseUrl}getApiKey(){return this.apiKey}getOrganizationId(){return this.organizationId}async request(R,f,q){let $=await fetch(f,{method:R,headers:{"Content-Type":"application/json",Authorization:`Bearer ${this.apiKey}`,"X-Organization-Id":this.organizationId},body:q!==void 0?JSON.stringify(q):void 0});if(!$.ok){let D=await $.text();throw Error(`HTTP ${$.status}: ${D}`)}return $.json()}async postOtelV1traces(){let R=this.baseUrl+"/otel/v1/traces";return this.request("POST",R,{})}async postOtelV1offlineTraces(){let R=this.baseUrl+"/otel/v1/offline-traces";return this.request("POST",R,{})}async postV1projectsResolve(R){let f=this.baseUrl+"/v1/projects/resolve/";return this.request("POST",f,R)}async postV1projects(R){let f=this.baseUrl+"/v1/projects";return this.request("POST",f,R)}async deleteV1projects(R){let f=this.baseUrl+`/v1/projects/${R}`;return this.request("DELETE",f,{})}async postV1projectsDatasets(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("POST",q,f)}async getV1projectsDatasets(R){let f=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("GET",f,void 0)}async getV1projectsDatasetsByDatasetIdentifierVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsDatasetsByDatasetIdentifierPage(R,f,q,$,D,P){let U=new URLSearchParams;if(q!==void 0)U.set("version",q);if($!==void 0)U.set("limit",$);if(D!==void 0)U.set("cursor_created_at",D);if(P!==void 0)U.set("cursor_example_id",P);let E=this.baseUrl+`/v1/projects/${R}/datasets/${f}/page`+(U.toString()?"?"+U.toString():"");return this.request("GET",E,void 0)}async postV1projectsDatasetsByDatasetIdentifierExamples(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/datasets/${f}/examples`;return this.request("POST",$,q)}async deleteV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("DELETE",q,{})}async getV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("GET",q,void 0)}async getV1projectsTestConfigs(R,f){let q=new URLSearchParams;if(f!==void 0)q.set("dataset_id",f);let $=this.baseUrl+`/v1/projects/${R}/test-configs`+(q.toString()?"?"+q.toString():"");return this.request("GET",$,void 0)}async postV1projectsTestConfigs(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs`;return this.request("POST",q,f)}async getV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestConfigsByTestConfigId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("PATCH",$,q)}async deleteV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("DELETE",q,{})}async getV1projectsTestRuns(R,f,q,$,D){let P=new URLSearchParams;if(f!==void 0)P.set("test_config_id",f);if(q!==void 0)P.set("dataset_id",q);if($!==void 0)P.set("status",$);if(D!==void 0)P.set("limit",D);let U=this.baseUrl+`/v1/projects/${R}/test-runs`+(P.toString()?"?"+P.toString():"");return this.request("GET",U,void 0)}async postV1projectsTestRuns(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs`;return this.request("POST",q,f)}async getV1projectsTestRunsByTestRunId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestRunsByTestRunIdStatus(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/status`;return this.request("PATCH",$,q)}async patchV1projectsTestRunsByTestRunIdSuccess(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/success`;return this.request("PATCH",$,q)}async getV1projectsTestRunsByTestRunIdItems(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/items`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdResults(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/results`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdGraph(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/graph`;return this.request("GET",q,void 0)}async getV1projectsExperimentsByRunId(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/experiments/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsEvalResults(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results`;return this.request("POST",q,f)}async postV1projectsEvalResultsExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/traces`;return this.request("POST",q,f)}async postV1projectsEvalQueueExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/examples`;return this.request("POST",q,f)}async postV1projectsEvalQueueTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/traces`;return this.request("POST",q,f)}async getV1projectsPromptsByName(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("commit_id",q);if($!==void 0)D.set("tag",$);let P=this.baseUrl+`/v1/projects/${R}/prompts/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsPrompts(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts`;return this.request("POST",q,f)}async postV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("POST",$,q)}async deleteV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("DELETE",$,q)}async getV1projectsPromptsByNameVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsScorers(R,f,q){let $=new URLSearchParams;if(f!==void 0)$.set("names",f);if(q!==void 0)$.set("is_trace",q);let D=this.baseUrl+`/v1/projects/${R}/scorers`+($.toString()?"?"+$.toString():"");return this.request("GET",D,void 0)}async getV1projectsScorersByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsScorersCustom(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom`;return this.request("POST",f,{})}async postV1projectsScorersCustomBundle(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom/bundle`;return this.request("POST",f,{})}async getV1projectsScorersCustomByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/custom/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsTracesByTraceIdTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/traces/${f}/tags`;return this.request("POST",$,q)}async getV1e2eFetchTraceByProjectNameByTraceId(R,f){let q=this.baseUrl+`/v1/e2e_fetch_trace/${R}/${f}`;return this.request("GET",q,void 0)}async getV1e2eTracesPerProject(R){let f=this.baseUrl+`/v1/e2e_traces_per_project/${R}`;return this.request("GET",f,void 0)}async postV1e2eFetchSpanScore(R){let f=this.baseUrl+"/v1/e2e_fetch_span_score/";return this.request("POST",f,R)}async postV1projectsJudges(R,f){let q=this.baseUrl+`/v1/projects/${R}/judges`;return this.request("POST",q,f)}async patchV1projectsJudgesByJudgeId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/judges/${f}`;return this.request("PATCH",$,q)}}var yR=()=>{};var F;var j=H(()=>{FR();F=class F{static RESET="\x1B[0m";static RED="\x1B[31m";static YELLOW="\x1B[33m";static GRAY="\x1B[90m";static Level={DEBUG:0,INFO:1,WARNING:2,ERROR:3,CRITICAL:4};static initialized=!1;static levelSetManually=!1;static currentLevel=F.Level.WARNING;static useColor=!0;static initialize(){if(!F.initialized){let R=process.env.JUDGMENT_NO_COLOR;if(F.useColor=!R&&process.stdout.isTTY,!F.levelSetManually){let f=m1.toLowerCase();if(f){let q={debug:F.Level.DEBUG,info:F.Level.INFO,warning:F.Level.WARNING,warn:F.Level.WARNING,error:F.Level.ERROR,critical:F.Level.CRITICAL};F.currentLevel=q[f]??F.Level.WARNING}}F.initialized=!0}}static setLevel(R){F.currentLevel=R,F.levelSetManually=!0}static setUseColor(R){F.useColor=R}static log(R,f){if(F.initialize(),R<F.currentLevel)return;let q=new Date().toISOString().replace("T"," ").substring(0,19),$=Object.keys(F.Level).find((U)=>F.Level[U]===R)??"UNKNOWN",D=`${q} - judgeval - ${$} - ${f}`;if(F.useColor)D=`${R===F.Level.DEBUG||R===F.Level.INFO?F.GRAY:R===F.Level.WARNING?F.YELLOW:F.RED}${D}${F.RESET}`;(R>=F.Level.ERROR?process.stderr:process.stdout).write(D+`
2
- `)}static debug(R){F.log(F.Level.DEBUG,R)}static info(R){F.log(F.Level.INFO,R)}static warning(R){F.log(F.Level.WARNING,R)}static warn(R){F.log(F.Level.WARNING,R)}static error(R){F.log(F.Level.ERROR,R)}static critical(R){F.log(F.Level.CRITICAL,R)}}});function LR(R){return new Promise((f)=>setTimeout(f,R))}async function k1(R,f={}){let{maxRetries:q=3,backoff:$=()=>1000,onRetry:D}=f;for(let P=1;P<=q;P++)try{return await R()}catch(U){if(P===q)throw U;D?.(P,U),await LR($(P))}throw Error("retry: exhausted all attempts")}var y1=()=>{};async function $R(R,f){let q=`org:${R.getOrganizationId()}:project:${f}`,$=L1.get(q);if($)return $;let D=f1.get(q);if(D)return D;let P=(async()=>{F.info(`Resolving project ID for project: ${f}`);let U=await k1(async()=>{let O=(await R.postV1projectsResolve({project_name:f})).project_id;if(!O)throw Error(`Project ID not found for project: ${f}`);return O},{maxRetries:3,backoff:(E)=>E*1000,onRetry:(E,O)=>{F.warning(`Failed to resolve project ID for '${f}' (attempt ${E}): ${String(O)}`)}});return F.info(`Resolved project ID: ${U}`),L1.set(q,U),U})();f1.set(q,P);try{return await P}finally{f1.delete(q)}}var L1,f1;var NR=H(()=>{j();y1();L1=new Map,f1=new Map});var YR=If((U$,q1)=>{var BR=process||{},N1=BR.argv||[],AR=BR.env||{},lf=!(!!AR.NO_COLOR||N1.includes("--no-color"))&&(!!AR.FORCE_COLOR||N1.includes("--color")||BR.platform==="win32"||(BR.stdout||{}).isTTY&&AR.TERM!=="dumb"||!!AR.CI),pf=(R,f,q=R)=>($)=>{let D=""+$,P=D.indexOf(f,R.length);return~P?R+df(D,f,q,P)+f:R+D+f},df=(R,f,q,$)=>{let D="",P=0;do D+=R.substring(P,$)+q,P=$+f.length,$=R.indexOf(f,P);while(~$);return D+R.substring(P)},A1=(R=lf)=>{let f=R?pf:()=>String;return{isColorSupported:R,reset:f("\x1B[0m","\x1B[0m"),bold:f("\x1B[1m","\x1B[22m","\x1B[22m\x1B[1m"),dim:f("\x1B[2m","\x1B[22m","\x1B[22m\x1B[2m"),italic:f("\x1B[3m","\x1B[23m"),underline:f("\x1B[4m","\x1B[24m"),inverse:f("\x1B[7m","\x1B[27m"),hidden:f("\x1B[8m","\x1B[28m"),strikethrough:f("\x1B[9m","\x1B[29m"),black:f("\x1B[30m","\x1B[39m"),red:f("\x1B[31m","\x1B[39m"),green:f("\x1B[32m","\x1B[39m"),yellow:f("\x1B[33m","\x1B[39m"),blue:f("\x1B[34m","\x1B[39m"),magenta:f("\x1B[35m","\x1B[39m"),cyan:f("\x1B[36m","\x1B[39m"),white:f("\x1B[37m","\x1B[39m"),gray:f("\x1B[90m","\x1B[39m"),bgBlack:f("\x1B[40m","\x1B[49m"),bgRed:f("\x1B[41m","\x1B[49m"),bgGreen:f("\x1B[42m","\x1B[49m"),bgYellow:f("\x1B[43m","\x1B[49m"),bgBlue:f("\x1B[44m","\x1B[49m"),bgMagenta:f("\x1B[45m","\x1B[49m"),bgCyan:f("\x1B[46m","\x1B[49m"),bgWhite:f("\x1B[47m","\x1B[49m"),blackBright:f("\x1B[90m","\x1B[39m"),redBright:f("\x1B[91m","\x1B[39m"),greenBright:f("\x1B[92m","\x1B[39m"),yellowBright:f("\x1B[93m","\x1B[39m"),blueBright:f("\x1B[94m","\x1B[39m"),magentaBright:f("\x1B[95m","\x1B[39m"),cyanBright:f("\x1B[96m","\x1B[39m"),whiteBright:f("\x1B[97m","\x1B[39m"),bgBlackBright:f("\x1B[100m","\x1B[49m"),bgRedBright:f("\x1B[101m","\x1B[49m"),bgGreenBright:f("\x1B[102m","\x1B[49m"),bgYellowBright:f("\x1B[103m","\x1B[49m"),bgBlueBright:f("\x1B[104m","\x1B[49m"),bgMagentaBright:f("\x1B[105m","\x1B[49m"),bgCyanBright:f("\x1B[106m","\x1B[49m"),bgWhiteBright:f("\x1B[107m","\x1B[49m")}};q1.exports=A1();q1.exports.createColors=A1});var m;var HR=H(()=>{m=class m{exampleId;createdAt;name;_properties;constructor(R,f,q,$){this.exampleId=R,this.createdAt=f,this.name=q,this._properties=$}static create(R={}){return new m(crypto.randomUUID(),new Date().toISOString(),null,{...R})}static META_KEYS=new Set(["example_id","created_at","name","trace_id","span_id","offline_trace_id","agent_offline_trace_id"]);static from(R){let f={};for(let q of Object.keys(R))if(!m.META_KEYS.has(q))f[q]=R[q];return new m(R.example_id??"",R.created_at??"",R.name??null,f)}get(R){return this._properties[R]}has(R){return R in this._properties}get properties(){return{...this._properties}}toJSON(){let R={example_id:this.exampleId,created_at:this.createdAt,name:this.name};for(let[f,q]of Object.entries(this._properties))R[f]=q;return R}}});function tf(){let R=new WeakSet;return function(f,q){if(typeof q==="bigint")return q.toString();if(typeof q==="object"&&q!==null){if(R.has(q))return"[Circular]";R.add(q)}return q}}function w(R){try{let f=JSON.stringify(R);if(typeof f==="string")return f;return String(f)}catch{try{let f=JSON.stringify(R,tf());return typeof f==="string"?f:String(R)}catch(f){return F.error(`safeStringify failed: ${f}`),String(R)}}}function n(R,f){if(typeof R==="string"||typeof R==="number"||typeof R==="boolean")return R;return f(R)}var N=H(()=>{j()});var I1="1.1.0";var v1=()=>{};var lR;var U1=H(()=>{v1();lR=I1});var UR=()=>{};function S(R,f,q){try{return f()}catch($){let D=$ instanceof Error&&$.stack?`
3
- ${$.stack}`:"";return F.error(`[Caught] An exception was raised in ${R}: ${String($)}${D}`),q}}var k=H(()=>{j()});var K1=H(()=>{k()});function A(R,f={}){let{pre:q,post:$,error:D,finally:P}=f;return async function(...E){let O=q?S("immutableWrapAsync.pre",()=>q(...E)):void 0,Q;try{let Y=await R.apply(this,E);if($)Q=S("immutableWrapAsync.post",()=>$(O,Y,E));return Y}catch(Y){if(D)Q=S("immutableWrapAsync.error",()=>D(O,Y,E));throw Y}finally{if(P)S("immutableWrapAsync.finally",()=>{P(Q)})}}}var u1=H(()=>{k()});var l1=H(()=>{k()});function O1(R,f={}){let{pre:q,yield:$,post:D,error:P,finally:U}=f;return async function*(...O){let Q=q?S("immutableWrapAsyncIterator.pre",()=>q(...O)):void 0,Y;try{for await(let W of R(...O)){if($)S("immutableWrapAsyncIterator.yield",()=>{$(Q,W)});yield W}if(D)Y=S("immutableWrapAsyncIterator.post",()=>D(Q))}catch(W){if(P)Y=S("immutableWrapAsyncIterator.error",()=>P(Q,W));throw W}finally{if(U)S("immutableWrapAsyncIterator.finally",()=>{U(Y)})}}}var E1=H(()=>{k()});function g(R,f){let q=R[Symbol.asyncIterator].bind(R),$=O1(()=>({[Symbol.asyncIterator]:q}),{yield:(D,P)=>{f.onYield(P)},post:()=>{f.onDone()},error:(D,P)=>{f.onError(P)},finally:()=>{f.onFinally()}});R[Symbol.asyncIterator]=()=>$()}var p1=H(()=>{E1()});var CR=H(()=>{K1();u1();l1();E1();p1()});function ZR(R,f){S("recordChatUsage",()=>{let q=f.prompt_tokens_details?.cached_tokens??0,$=f.prompt_tokens+f.completion_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.prompt_tokens-q:f.prompt_tokens,output_tokens:f.completion_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}var V1=H(()=>{UR();K();k();N()});function d1(R){R.chat.completions.parse=A(R.chat.completions.parse.bind(R.chat.completions),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),q},post:(f,q)=>{if(!f)return;if(V.setOutput(w(q),f),q.usage)ZR(f,q.usage);return V.recordLLMMetadata({model:q.model},f),f},error:(f,q)=>{if(f)V.setError(q,f);return f},finally:(f)=>{f?.end()}})}var s1=H(()=>{K();N();CR();V1()});function n1(R){R.chat.completions.create=A(R.chat.completions.create.bind(R.chat.completions),{pre:(f)=>{if(f.stream)f.stream_options??={include_usage:!0};let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,E="";return g(U,{onYield(O){if(typeof O.choices[0]?.delta.content==="string")E+=O.choices[0].delta.content;if(O.usage)ZR(D,O.usage)},onDone(){V.setOutput(E,D)},onError(O){V.setError(O,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)ZR(D,P.usage);return V.recordLLMMetadata({model:P.model},D),f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var g1=H(()=>{K();N();CR();V1()});function i1(R,f){S("images.recordUsage",()=>{let q="input_tokens_details"in f?f.input_tokens_details:void 0,$=q?.image_tokens??0;if(V.recordLLMMetadata({non_cached_input_tokens:q?.text_tokens??0,output_tokens:f.output_tokens||void 0},R),$)V.setAttribute("judgment.usage.non_cached_input_image_tokens",$,R);if(f.output_tokens)V.setAttribute("judgment.usage.output_image_tokens",f.output_tokens,R);V.setAttribute("judgment.usage.metadata",w(f),R)})}function o1(R){R.images.generate=A(R.images.generate.bind(R.images),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,E;return g(U,{onYield(O){if(rf.has(O.type))E=O,i1(D,E.usage)},onDone(){V.setOutput(w(E??{}),D)},onError(O){V.setError(O,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)i1(D,P.usage);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var rf;var t1=H(()=>{UR();K();k();N();CR();rf=new Set(["image_generation.completed","image_edit.completed"])});function c1(R,f){S("responses.recordUsage",()=>{let q=f.input_tokens_details.cached_tokens,$=f.input_tokens+f.output_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.input_tokens-q:f.input_tokens,output_tokens:f.output_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}function r1(R){R.responses.create=A(R.responses.create.bind(R.responses),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,E="";return g(U,{onYield(O){if(O.type==="response.output_text.delta")E+=O.delta;if(O.type==="response.completed"){let Q=O.response;if(Q.usage)c1(D,Q.usage);V.recordLLMMetadata({model:Q.model},D)}},onDone(){V.setOutput(E,D)},onError(O){V.setError(O,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)c1(D,P.usage);if(typeof P.model==="string")V.recordLLMMetadata({model:P.model},D);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var e1=H(()=>{UR();K();k();N();CR()});function F1(R){return S("wrapOpenAI",()=>{n1(R),d1(R),r1(R),o1(R)}),R}var a1=H(()=>{k();s1();g1();t1();e1()});function Q1(R){return F1(R)}var Y1=H(()=>{a1()});function $q(R){return Function.prototype.toString.call(R)}function Dq(R){let f=$q(R).replace(qq,"");return f.match(ef)||f.match(af)}function Rf(R){let f=Dq(R);if(!f||!f[1])return[];return f[1].split(Rq).map((q)=>{return q.replace(fq,(D,P,U)=>U).trim()}).filter((q)=>q.length>0)}var ef,af,Rq,fq,qq;var ff=H(()=>{ef=/^([^(]+?)=>/,af=/^[^(]*\(\s*([^)]*)\)/m,Rq=/,/,fq=/^\s*(_?)(\S+?)\1\s*$/,qq=/((\/\/.*$)|(\/\*[\s\S]*?\*\/))/gm});import{ROOT_CONTEXT as Pq,context as i}from"@opentelemetry/api";import{AsyncLocalStorage as $f}from"async_hooks";function H1(){return Df.getStore()===!0}function Pf(R){if(W1=R,qf)return;let f=i;f.active=()=>{if(!H1())return Uq();let q=dR.getStore();if(q)return q;return W1?W1():Pq},f.with=(q,$,D,...P)=>{if(!H1())return Oq(q,$,D,...P);return dR.run(q,()=>$.apply(D,P))},f.bind=(q,$)=>{if(!H1())return Eq(q,$);if(typeof $!=="function")return $;let D=$;return(...P)=>dR.run(q,()=>D(...P))},qf=!0}function S1(R,f){return Df.run(!0,()=>dR.run(R,f))}var qf=!1,W1=null,Df,dR,Uq,Oq,Eq;var Uf=H(()=>{Df=new $f,dR=new $f,Uq=i.active.bind(i),Oq=i.with.bind(i),Eq=i.bind.bind(i)});import{INVALID_SPAN_CONTEXT as Vq,ROOT_CONTEXT as Fq,SpanStatusCode as Of,trace as o}from"@opentelemetry/api";import{registerInstrumentations as Qq}from"@opentelemetry/instrumentation";import{AsyncLocalStorage as Yq}from"async_hooks";class Ef{_provider;constructor(R){this._provider=R}startSpan(R,f,q){let $=q??this._provider.getCurrentContext();return this._provider._getDelegateTracer().startSpan(R,f,$)}startActiveSpan(R,...f){let q={},$=this._provider.getCurrentContext(),D;if(f.length===1)D=f[0];else if(f.length===2)q=f[0],D=f[1];else q=f[0],$=f[1],D=f[2];let P=this.startSpan(R,q,$);return this._provider.useSpan(P,!1,!1,!1,()=>D(P))}}class Vf{startSpan(){return o.wrapSpanContext(Vq)}startActiveSpan(R,...f){return(f.length===1?f[0]:f.length===2?f[1]:f[2])(this.startSpan())}}class G{static _instance=null;_activeTracer=null;_instrumentations=[];_noOpTracer;_proxyTracer;_tracers=new Set;constructor(){this._noOpTracer=new Vf,this._proxyTracer=new Ef(this),Pf(()=>this.getCurrentContext())}static getInstance(){return G._instance??=new G,G._instance}static installAsGlobalTracerProvider(){let R=G.getInstance();return o.setGlobalTracerProvider(R)}register(R){this._tracers.add(R)}deregister(R){this._tracers.delete(R)}setActive(R){let f=this.getCurrentSpan();if(f?.isRecording()){if(o.getSpan(this.getCurrentContext())===f)return F.error("Cannot set_active() while a root span is active. Keeping existing tracer provider."),!1}return this.register(R),this._activeTracer=R,!0}getActiveTracer(){return this._activeTracer}getCurrentContext(){return sR.getStore()??Fq}setSpan(R,f){return o.setSpan(R,f)}wrapSpanContext(R){return o.wrapSpanContext(R)}getCurrentSpan(){let R=this.getCurrentContext();return o.getSpan(R)}hasActiveRootSpan(){if(!this.getCurrentSpan()?.isRecording())return!1;return!0}_getDelegateTracer(){let R=this._activeTracer;if(!R)return F.debug("No active tracer, returning NoOpTracer"),this._noOpTracer;return R._tracerProvider.getTracer(Wq)}getTracer(R,f,q){return this._proxyTracer}addInstrumentation(R){try{Qq({tracerProvider:this,instrumentations:[R]}),this._instrumentations.push(R)}catch(f){F.error(`Failed to add instrumentation: ${String(f)}`)}}useSpan(R,f,q,$,D){let P=this.getCurrentContext(),U=o.setSpan(P,R);return sR.run(U,()=>S1(U,()=>{try{let E=D();if(E instanceof Promise)return E.catch((O)=>{if(R.isRecording()){if(q)R.recordException(O);if($){let Q=O;R.setStatus({code:Of.ERROR,message:`${Q.name}: ${Q.message}`})}}throw O}).finally(()=>{if(f)R.end()});if(f)R.end();return E}catch(E){if(R.isRecording()){if(q)R.recordException(E);if($){let O=E;R.setStatus({code:Of.ERROR,message:`${O.name}: ${O.message}`})}}if(f)R.end();throw E}}))}attachContext(R){sR.enterWith(R)}withContext(R,f){return sR.run(R,()=>S1(R,f))}async forceFlush(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.forceFlush()));for(let f of R)if(f.status==="rejected")F.error(`forceFlush failed: ${String(f.reason)}`)}async shutdown(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.shutdown()));for(let f of R)if(f.status==="rejected")F.error(`shutdown failed: ${String(f.reason)}`);this._activeTracer=null,this._tracers.clear()}}var Wq="judgeval",sR;var B=H(()=>{j();Uf();sR=new Yq});var Ff="=",nR=";",hR=",",gR="baggage",Qf=180,Yf=4096,Wf=8192;import{baggageEntryMetadataFromString as Hq}from"@opentelemetry/api";function Hf(R){return R.reduce((f,q)=>{let $=`${f}${f!==""?hR:""}${q}`;return $.length>Wf?f:$},"")}function Sf(R){return R.getAllEntries().map(([f,q])=>{let $=`${encodeURIComponent(f)}=${encodeURIComponent(q.value)}`;if(q.metadata!==void 0)$+=nR+q.metadata.toString();return $})}function Cf(R){let f=R.split(nR);if(f.length<=0)return;let q=f.shift();if(!q)return;let $=q.indexOf(Ff);if($<=0)return;let D=decodeURIComponent(q.substring(0,$).trim()),P=decodeURIComponent(q.substring($+1).trim()),U;if(f.length>0)U=Hq(f.join(nR));return{key:D,value:P,metadata:U}}var Zf=()=>{};import{isTracingSuppressed as Sq}from"@opentelemetry/core";class t{inject(R,f,q){let $=u(R);if(!$||Sq(R))return;let D=Sf($).filter((U)=>U.length<=Yf).slice(0,Qf),P=Hf(D);if(P.length>0)q.set(f,gR,P)}extract(R,f,q){let $=q.get(f,gR),D=Array.isArray($)?$.join(hR):$;if(!D)return R;let P={};if(D.length===0)return R;if(D.split(hR).forEach((E)=>{let O=Cf(E);if(O){let Q={value:O.value};if(O.metadata)Q.metadata=O.metadata;P[O.key]=Q}}),Object.entries(P).length===0)return R;return GR(R,XR(P))}fields(){return[gR]}}var iR=H(()=>{JR();Zf()});var Z1={};R1(Z1,{setBaggage:()=>GR,getBaggage:()=>u,getActiveBaggage:()=>Zq,deleteBaggage:()=>hq,createBaggage:()=>XR,baggageEntryMetadataFromString:()=>Xq,JudgmentBaggagePropagator:()=>t});import{createContextKey as Cq,propagation as hf}from"@opentelemetry/api";import{baggageEntryMetadataFromString as Xq}from"@opentelemetry/api";function u(R){return R.getValue(C1)}function Zq(){return u(G.getInstance().getCurrentContext())}function GR(R,f){return R.setValue(C1,f)}function hq(R){return R.deleteValue(C1)}var XR,C1;var JR=H(()=>{B();iR();XR=hf.createBaggage.bind(hf),C1=Cq("baggage")});var G1={};R1(G1,{setGlobalTextmap:()=>_q,inject:()=>Mq,getGlobalTextmap:()=>h1,extract:()=>X1});import{defaultTextMapGetter as Gq,defaultTextMapSetter as Jq}from"@opentelemetry/api";import{CompositePropagator as bq,W3CTraceContextPropagator as wq}from"@opentelemetry/core";function h1(){return Xf}function _q(R){Xf=R}function Gf(R){if(R!==void 0)return R;return G.getInstance().getCurrentContext()}function Mq(R,f,q=Jq){S("propagation.inject",()=>{h1().inject(Gf(f),R,q)})}function X1(R,f,q=Gq){let $=Gf(f);return S("propagation.extract",()=>h1().extract($,R,q),$)}var Xf;var J1=H(()=>{k();iR();B();Xf=new bq({propagators:[new wq,new t]})});import{INVALID_SPAN_CONTEXT as zq,SpanStatusCode as OR}from"@opentelemetry/api";import{randomUUID as jq}from"crypto";class V{projectName;projectId;apiKey;organizationId;apiUrl;environment;serializer;_tracerProvider;_client;_enableMonitoring;supportsLiveInstrumentation=!0;constructor(R,f,q,$,D,P,U,E,O,Q){this.projectName=R,this.projectId=f,this.apiKey=q,this.organizationId=$,this.apiUrl=D,this.environment=P,this.serializer=U,this._tracerProvider=E,this._client=O,this._enableMonitoring=Q}setActive(){return G.getInstance().setActive(this)}static _getProxyProvider(){return G.getInstance()}static _getSerializer(){return V._getProxyProvider().getActiveTracer()?.serializer??w}static _getCurrentTraceAndSpanId(){let f=V._getProxyProvider().getCurrentSpan();if(!f?.isRecording())return null;let q=f.spanContext();if(!q.traceId||!(q.traceFlags&1))return null;return[q.traceId,q.spanId]}static _emitPartial(){S("BaseTracer._emitPartial",()=>{let R=V._getProxyProvider().getActiveTracer();if(!R||!R.supportsLiveInstrumentation)return;R.getSpanProcessor().emitPartial()})}static getCurrentSpan(){return V._getProxyProvider().getCurrentSpan()}static async forceFlush(){await V._getProxyProvider().forceFlush()}static async shutdown(){await V._getProxyProvider().shutdown()}static registerOTELInstrumentation(R){S("BaseTracer.registerOTELInstrumentation",()=>{V._getProxyProvider().addInstrumentation(R)})}static wrap(R){return Q1(R)}static getOTELTracer(){return V._getProxyProvider().getTracer(Jf)}static startSpan(R,f){let q=V.getOTELTracer().startSpan(R,{attributes:f});return V._emitPartial(),q}static startActiveSpan(R,f){let{name:q,attributes:$}=R;return V.getOTELTracer().startActiveSpan(q,{attributes:$},(D)=>{V._emitPartial();try{let P=f(D);if(P instanceof Promise)return P.finally(()=>{D.end()});return D.end(),P}catch(P){throw D.end(),P}})}static span(R,f){return V.startActiveSpan({name:R},(q)=>{try{let $=f(q);if($ instanceof Promise)return $.catch((D)=>{throw q.setStatus({code:OR.ERROR,message:String(D)}),q.recordException(D),D});return $}catch($){throw q.setStatus({code:OR.ERROR,message:String($)}),q.recordException($),$}})}static with(R,f){return V.span(R,f)}static continueTrace(R,f){let q=V._getProxyProvider(),$=X1(R);return q.withContext($,()=>f($))}static observe(R,f){let q;if(typeof R==="function")q=R;else f=R;let{spanType:$="span",spanName:D,recordInput:P=!0,recordOutput:U=!0,fork:E=!1}=f??{},O=V._getProxyProvider(),Q=(Y)=>{let W=D??Y.name;return function(...C){let J=O.getTracer(Jf);if(E&&O.getActiveTracer()!==null&&O.getCurrentSpan()?.isRecording()===!0){let h=V._getSerializer(),Z=J.startSpan(W),T=Z.spanContext();if($)Z.setAttribute("judgment.span_kind",$);let xR={"judgment.link.source_trace_id":T.traceId,"judgment.link.source_span_id":T.spanId};if($)xR["judgment.span_kind"]=$;let mR=O.setSpan(O.getCurrentContext(),O.wrapSpanContext(zq)),L=J.startSpan(W,{attributes:xR},mR),p=L.spanContext();Z.setAttribute("judgment.link.target_trace_id",p.traceId),Z.setAttribute("judgment.link.target_span_id",p.spanId);let a=()=>{L.end(),Z.end()},j1=(z)=>{for(let y of[L,Z])y.recordException(z),y.setStatus({code:OR.ERROR,message:String(z)})},x1=(z)=>{let y=n(z,h);L.setAttribute("judgment.output",y),Z.setAttribute("judgment.output",y)};if(P){let z=n(bf(Y,C),h);L.setAttribute("judgment.input",z),Z.setAttribute("judgment.input",z)}return V._emitPartial(),O.useSpan(L,!1,!1,!1,()=>{try{let z=Y.call(this,...C);if(z instanceof Promise)return z.then((y)=>{if(U)x1(y);return y}).catch((y)=>{throw j1(y),y}).finally(a);if(U)x1(z);return a(),z}catch(z){throw j1(z),a(),z}})}return J.startActiveSpan(W,(h)=>{if($)h.setAttribute("judgment.span_kind",$);try{if(P)h.setAttribute("judgment.input",n(bf(Y,C),V._getSerializer()));V._emitPartial();let Z=Y.call(this,...C);if(Z instanceof Promise)return Z.then((T)=>{if(U)h.setAttribute("judgment.output",n(T,V._getSerializer()));return T}).catch((T)=>{throw h.recordException(T),h.setStatus({code:OR.ERROR,message:String(T)}),T}).finally(()=>{h.end()});if(U)h.setAttribute("judgment.output",n(Z,V._getSerializer()));return h.end(),Z}catch(Z){throw h.recordException(Z),h.setStatus({code:OR.ERROR,message:String(Z)}),h.end(),Z}})}};if(!q)return Q;return Q(q)}static _resolveSpan(R){if(R)return R;return V._getProxyProvider().getCurrentSpan()}static setSpanKind(R,f){S("BaseTracer.setSpanKind",()=>{if(!R)return;let q=V._resolveSpan(f);if(q?.isRecording())q.setAttribute("judgment.span_kind",R)})}static setLLMSpan(){V.setSpanKind("llm")}static setToolSpan(){V.setSpanKind("tool")}static setGeneralSpan(){V.setSpanKind("span")}static setAttribute(R,f,q){S("BaseTracer.setAttribute",()=>{let $=V._resolveSpan(q);if(!$?.isRecording())return;if(!R||f==null)return;$.setAttribute(R,n(f,V._getSerializer()))})}static setAttributes(R,f){for(let[q,$]of Object.entries(R))if(f)V.setAttribute(q,$,f);else V.setAttribute(q,$)}static setInput(R,f){if(f)V.setAttribute("judgment.input",R,f);else V.setAttribute("judgment.input",R)}static setOutput(R,f){if(f)V.setAttribute("judgment.output",R,f);else V.setAttribute("judgment.output",R)}static setError(R,f){S("BaseTracer.setError",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;q.recordException(R),q.setStatus({code:OR.ERROR,message:String(R)})})}static recordLLMMetadata(R,f){S("BaseTracer.recordLLMMetadata",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;if(typeof R.model==="string")q.setAttribute("judgment.llm.model",R.model);if(typeof R.provider==="string")q.setAttribute("judgment.llm.provider",R.provider);if(typeof R.non_cached_input_tokens==="number")q.setAttribute("judgment.usage.non_cached_input_tokens",R.non_cached_input_tokens);if(typeof R.output_tokens==="number")q.setAttribute("judgment.usage.output_tokens",R.output_tokens);if(typeof R.cache_read_input_tokens==="number")q.setAttribute("judgment.usage.cache_read_input_tokens",R.cache_read_input_tokens);if(typeof R.cache_creation_input_tokens==="number")q.setAttribute("judgment.usage.cache_creation_input_tokens",R.cache_creation_input_tokens);if(typeof R.total_cost_usd==="number")q.setAttribute("judgment.usage.total_cost_usd",R.total_cost_usd)})}static _setPropagatingBaggageKey(R,f){S("BaseTracer._setPropagatingBaggageKey",()=>{let q=V._getProxyProvider(),$=q.getCurrentSpan();if(!$?.isRecording())return;$.setAttribute(R,f);let D=q.getCurrentContext(),P=(u(D)??XR()).setEntry(R,{value:f});q.attachContext(GR(D,P))})}static setCustomerId(R){V._setPropagatingBaggageKey("judgment.customer_id",R)}static setCustomerUserId(R){V._setPropagatingBaggageKey("judgment.customer_user_id",R)}static setSessionId(R){V._setPropagatingBaggageKey("judgment.session_id",R)}static tag(R){S("BaseTracer.tag",()=>{if(!R||Array.isArray(R)&&R.length===0)return;let q=V._getProxyProvider().getActiveTracer();if(!q?.projectId||!q._client)return;if(!q.supportsLiveInstrumentation)return;let $=V._getCurrentTraceAndSpanId();if(!$)return;let[D]=$,P=Array.isArray(R)?R:[R];q._client.postV1projectsTracesByTraceIdTags(q.projectId,D,{tags:P}).catch((U)=>{F.error(`tag failed: ${String(U)}`)})})}static asyncEvaluate(R,f){S("BaseTracer.asyncEvaluate",()=>{let{judge:q,example:$}=R,P=V._getProxyProvider().getActiveTracer();if(!P?.projectId)return;if(!P.supportsLiveInstrumentation)return;let U=V._resolveSpan(f);if(!U?.isRecording())return;let E=P.getSpanProcessor(),O=U.spanContext(),Q=E.stateIncr(O,"pending_evals_count"),Y={project_id:P.projectId,eval_name:`async_evaluate_${q}_${Q}`,judges:[{name:q}],examples:[{...$,example_id:jq(),created_at:new Date().toISOString(),trace_id:O.traceId,span_id:O.spanId}],is_offline:!1,is_behavior:!1},W=E.stateAppend(O,"pending_evals",Y);U.setAttribute("judgment.pending_trace_eval",JSON.stringify(W))})}}function bf(R,f){try{let q=Rf(R).map((D)=>D.replace(/^\.\.\./,"").split("=")[0].trim()).filter((D)=>D.length>0),$={};return q.forEach((D,P)=>{if(P<f.length)$[D]=f[P]}),$}catch{return{}}}var Jf="judgeval";var K=H(()=>{UR();Y1();ff();k();j();N();JR();B();J1()});import{OTLPTraceExporter as xq}from"@opentelemetry/exporter-trace-otlp-http";class v{_delegate;constructor(R,f,q,$){if(!R){this._delegate=null;return}this._delegate=new xq({url:R,headers:{Authorization:`Bearer ${f}`,"X-Organization-Id":q,"X-Project-Id":$}})}export(R,f){F.info(`Exported ${R.length} spans`),this._delegate?.export(R,f)}shutdown(){return this._delegate?.shutdown()??Promise.resolve()}forceFlush(){return this._delegate?.forceFlush()??Promise.resolve()}}var bR=H(()=>{j()});import{ExportResultCode as mq}from"@opentelemetry/core";var c;var oR=H(()=>{bR();c=class c extends v{constructor(){super("","","","")}export(R,f){f({code:mq.SUCCESS})}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}}});class wR{_keyPredicate;constructor(R=b1){this._keyPredicate=R}onStart(R,f){let q=u(f)?.getAllEntries()??[];for(let[$,D]of q)if(this._keyPredicate($))R.setAttribute($,D.value)}onEnd(R){}forceFlush(){return Promise.resolve()}shutdown(){return Promise.resolve()}}var b1=()=>!0;var w1=H(()=>{JR()});import{BatchSpanProcessor as kq}from"@opentelemetry/sdk-trace-base";function ER(R){return`${R.traceId}:${R.spanId}`}function yq(R){return R[0]===0&&R[1]===0}var l;var _R=H(()=>{UR();k();B();w1();l=class l extends kq{tracer;_state=new Map;_spanFinalizers;_baggageProcessor;constructor(R,f,q){super(f,q);this.tracer=R,this._spanFinalizers=new FinalizationRegistry(($)=>{this._cleanupSpanState($)}),this._baggageProcessor=new wR}_cleanupSpanState(R){this._state.delete(R)}_registerSpan(R){let f=R.spanContext();if(!f.traceId||!f.spanId)return;let q=ER(f);this._spanFinalizers.register(R,q)}stateSet(R,f,q){let $=ER(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);D.set(f,q)}stateGet(R,f,q){let $=ER(R),D=this._state.get($);if(!D?.has(f))return q;return D.get(f)}stateIncr(R,f){let q=ER(R),$=this._state.get(q);if(!$)$=new Map,this._state.set(q,$);let D=$.get(f),P=typeof D==="number"?D:0;return $.set(f,P+1),P}stateAppend(R,f,q){let $=ER(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);let P=D.get(f),U=Array.isArray(P)?[...P,q]:[q];return D.set(f,U),U}_emitSpan(R,f=!1){let q=R.spanContext();if(!q.traceId)return;let $=this.stateIncr(q,"judgment.update_id"),D={...R.attributes,"judgment.update_id":$};if(f)delete D["judgment.pending_trace_eval"];let P=Object.create(R);Object.defineProperty(P,"attributes",{value:D,writable:!1});let U=yq(R.endTime)?R.startTime:R.endTime;Object.defineProperty(P,"endTime",{value:U,writable:!1}),super.onEnd(P)}emitPartial(){S("JudgmentSpanProcessor.emitPartial",()=>{let f=G.getInstance().getCurrentSpan();if(!f?.isRecording())return;let q=f.spanContext();if(!q.traceId)return;if(this.stateGet(q,"disable_partial_emit",!1))return;this._emitSpan(f,!0)})}onStart(R,f){S("JudgmentSpanProcessor.onStart",()=>{this._baggageProcessor.onStart(R,f),this._registerSpan(R)})}onEnd(R){S("JudgmentSpanProcessor.onEnd",()=>{let f=R.spanContext();if(!f.traceId){super.onEnd(R);return}let q=ER(f);try{if(!this.stateGet(f,"cancelled",!1))this._emitSpan(R)}finally{this._cleanupSpanState(q)}})}}});var MR;var _1=H(()=>{oR();_R();MR=class MR extends l{constructor(){super(null,new c)}onStart(R,f){}onEnd(R){}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}emitPartial(){}stateSet(R,f,q){}stateGet(R,f,q){return q}stateIncr(R,f){return 0}stateAppend(R,f,q){return[q]}}});import{defaultResource as Lq,resourceFromAttributes as Nq}from"@opentelemetry/resources";import{NodeTracerProvider as wf}from"@opentelemetry/sdk-trace-node";var I;var tR=H(()=>{FR();yR();j();NR();N();U1();K();B();bR();oR();_R();_1();I=class I extends V{_spanExporter=null;_spanProcessor=null;constructor(R,f,q,$,D,P,U,E,O,Q){super(R,f,q,$,D,P,U,E,O,Q)}static async init(R={}){let f=R.apiKey??RR,q=R.organizationId??fR,$=R.apiUrl??qR,D=R.projectName??null,P=R.serializer??w,U=!0;if(!D)F.warning("project_name not provided. Tracer will not export spans."),U=!1;if(!f)F.warning("api_key not provided. Tracer will not export spans."),U=!1;if(!q)F.warning("organization_id not provided. Tracer will not export spans."),U=!1;if(!$)F.warning("api_url not provided. Tracer will not export spans."),U=!1;let E=null,O=null;if(U&&D&&f&&q&&$){if(E=new s($,f,q),O=await $R(E,D).catch(()=>null),!O)F.warning(`Project '${D}' not found. Tracer will not export spans.`),U=!1}let Q={"service.name":D??"unknown","telemetry.sdk.name":"judgeval","telemetry.sdk.version":lR};if(R.environment)Q["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(Q,R.resourceAttributes);let Y=Lq().merge(Nq(Q)),W=new wf({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits}),C=new I(D,O,f,q,$,R.environment??null,P,W,E,U);if(U){let b=new wf({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[C.getSpanProcessor(),...R.spanProcessors??[]]});C._tracerProvider=b}if(G.getInstance().register(C),R.setActive??!0)C.setActive();return C}getSpanExporter(){if(this._spanExporter)return this._spanExporter;if(!this._enableMonitoring||!this.projectId||!this.apiKey||!this.organizationId||!this.apiUrl)this._spanExporter=new c;else{let R=this.apiUrl.endsWith("/")?this.apiUrl+"otel/v1/traces":this.apiUrl+"/otel/v1/traces";this._spanExporter=new v(R,this.apiKey,this.organizationId,this.projectId)}return this._spanExporter}getSpanProcessor(){if(this._spanProcessor)return this._spanProcessor;if(!this._enableMonitoring)this._spanProcessor=new MR;else this._spanProcessor=new l(this,this.getSpanExporter());return this._spanProcessor}}});var TR;var M1=H(()=>{HR();_R();TR=class TR extends l{_dataset;_exampleFields;_seenTraceIds=new Set;constructor(R,f,q){super(R,f);this._dataset=q.dataset,this._exampleFields={...q.exampleFields??{}}}_maybeCreateExample(R){if(R.parentSpanContext)return;let f=R.spanContext();if(!f?.traceId)return;if(this._seenTraceIds.has(f.traceId))return;this._seenTraceIds.add(f.traceId);let q=m.create({...this._exampleFields,offline_trace_id:f.traceId});this._dataset.push(q)}onEnd(R){try{this._maybeCreateExample(R)}finally{super.onEnd(R)}}}});var Tf={};R1(Tf,{OfflineTracer:()=>r});import{defaultResource as Aq,resourceFromAttributes as Bq}from"@opentelemetry/resources";import{NodeTracerProvider as _f}from"@opentelemetry/sdk-trace-node";var Mf="otel/v1/offline-traces",r;var cR=H(()=>{FR();yR();NR();N();U1();B();tR();bR();M1();r=class r extends I{supportsLiveInstrumentation=!1;_offlineApiUrl;_offlineApiKey;_offlineOrganizationId;_offlineProjectId;_dataset;_exampleFields;_offlineSpanExporter=null;_offlineSpanProcessor=null;constructor(R){super(R.projectName,R.projectId,R.apiKey,R.organizationId,R.apiUrl,R.environment,R.serializer,R.tracerProvider,R.client,!0);this._offlineApiUrl=R.apiUrl,this._offlineApiKey=R.apiKey,this._offlineOrganizationId=R.organizationId,this._offlineProjectId=R.projectId,this._dataset=R.dataset,this._exampleFields=R.exampleFields}static async create(R){let f=R.apiKey??RR,q=R.organizationId??fR,$=R.apiUrl??qR,D=R.projectName,P=R.serializer??w;if(!D)throw Error("projectName is required for OfflineTracer");if(!f)throw Error("apiKey is required for OfflineTracer");if(!q)throw Error("organizationId is required for OfflineTracer");if(!$)throw Error("apiUrl is required for OfflineTracer");let U=new s($,f,q),E;try{E=await $R(U,D)}catch(J){throw Error(`Project '${D}' not found; cannot start OfflineTracer: ${String(J)}`)}let O={"service.name":D,"telemetry.sdk.name":"judgeval","telemetry.sdk.version":lR,"judgment.offline":"true"};if(R.environment)O["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(O,R.resourceAttributes);let Q=Aq().merge(Bq(O)),Y=new r({projectName:D,projectId:E,apiKey:f,organizationId:q,apiUrl:$,environment:R.environment??null,serializer:P,tracerProvider:new _f({resource:Q}),client:U,dataset:R.dataset,exampleFields:{...R.exampleFields??{}}}),W=new _f({resource:Q,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[Y.getSpanProcessor(),...R.spanProcessors??[]]});if(Y._tracerProvider=W,G.getInstance().register(Y),R.setActive??!0)Y.setActive();return Y}getSpanExporter(){if(this._offlineSpanExporter)return this._offlineSpanExporter;let R=this._offlineApiUrl.endsWith("/")?this._offlineApiUrl+Mf:this._offlineApiUrl+"/"+Mf;return this._offlineSpanExporter=new v(R,this._offlineApiKey,this._offlineOrganizationId,this._offlineProjectId),this._offlineSpanExporter}getSpanProcessor(){if(this._offlineSpanProcessor)return this._offlineSpanProcessor;return this._offlineSpanProcessor=new TR(this,this.getSpanExporter(),{dataset:this._dataset,exampleFields:this._exampleFields}),this._offlineSpanProcessor}}});FR();yR();NR();j();class QR{}j();var $1=VR(YR(),1);j();var X=VR(YR(),1),sf=2000;class DR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async _poll(R,f,q,$){let D=Date.now();while(!0){let P=(Date.now()-D)/1000;if(P>$)throw Error(`Evaluation timed out after ${$}s`);let U=await this._client.getV1projectsExperimentsByRunId(R,f),E=U.results??[];if(E.length>=q){let Q=U.ui_results_url??"Failed to get UI results URL";return console.log(`${X.default.green("✓")} Evals completed and saved in ${X.default.bold(`${P.toFixed(1)}s`)}`),{results:E,url:Q}}await new Promise((Q)=>setTimeout(Q,sf))}}_displayResults(R,f,q,$){let D=[],P=0,U=0;console.log();for(let E=0;E<f.length;E++){let O=f[E],Q=O.scorers.every((Y)=>Boolean(Y.success));if(Q)P++,console.log(`${X.default.green("✓")} Example ${E+1}: ${X.default.green("PASSED")}`);else U++,console.log(`${X.default.red("✗")} Example ${E+1}: ${X.default.red("FAILED")}`);for(let Y of O.scorers){let W=Y.score!==null?Y.score.toFixed(3):"N/A",C=Y.success?X.default.green(W):X.default.red(W);console.log(` ${X.default.dim(`${Y.name}:`)} ${C} ${X.default.dim(`(threshold: ${Y.threshold})`)}`)}D.push({success:Q,scorers:O.scorers,example:R[E]})}if(console.log(),P===D.length)console.log(`${X.default.bold(X.default.green("✓ All tests passed!"))} (${P}/${D.length})`);else console.log(`${X.default.bold(X.default.yellow("⚠ Results:"))} ${X.default.green(`${P} passed`)} | ${X.default.red(`${U} failed`)}`);if(console.log(`${X.default.dim("View full details:")} ${X.default.underline(q)}`),console.log(),$&&D.some((E)=>!E.success)){let E=[`Evaluation failed: ${U}/${D.length} examples failed`];for(let O=0;O<D.length;O++)if(!D[O].success){E.push(` Example ${O+1}:`);for(let Q of D[O].scorers)if(!Q.success){if(E.push(` ${Q.name}: ${Q.score!==null?Q.score.toFixed(3):"N/A"} (threshold: ${Q.threshold})`),Q.reason)E.push(` ${Q.reason}`)}}throw Error(E.join(`
4
- `))}return D}async run(R,f,q,$=!1,D=300){if(!this._projectId)return F.error("Project ID is not resolved. Evaluation requires a valid project."),[];let P=this._projectId,U=crypto.randomUUID(),E=new Date().toISOString();console.log(),console.log(X.default.bold(X.default.cyan("Starting Evaluation"))),console.log(`${X.default.dim("Run:")} ${q}`),console.log(`${X.default.dim("Project:")} ${this._projectName}`),console.log(`${X.default.dim("Examples:")} ${R.length} | ${X.default.dim("Scorers:")} ${f.length}`),console.log();let O=this._buildPayload(U,P,q,E,R,f),Q=await this._submit(P,U,R,f,O),{results:Y,url:W}=await this._poll(P,U,Q,D);return this._displayResults(R,Y,W,$)}}class vR extends DR{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:[],custom_scorers:[]}}async _submit(R,f,q,$,D){let P=Date.now(),U=q.flatMap((W,C)=>$.map((J)=>J.score(W).then((b)=>({exampleIdx:C,scorer:J,result:b,error:null})).catch((b)=>({exampleIdx:C,scorer:J,result:null,error:String(b)})))),E=await Promise.all(U),O=((Date.now()-P)/1000).toFixed(1);console.log(`${$1.default.green("✓")} Scoring completed in ${$1.default.bold(`${O}s`)}`);let Q=new Map;for(let W of E){let C=Q.get(W.exampleIdx);if(!C)C=[],Q.set(W.exampleIdx,C);C.push(W)}let Y=q.map((W,C)=>{return{scorers_data:(Q.get(C)??[]).map((b)=>{if(b.error!==null)return{scorer_name:b.scorer.constructor.name,value:0,reason:"",error:b.error};let h=b.result;return{scorer_name:b.scorer.constructor.name,value:h.value,reason:h.reason,...h.citations&&{citations:h.citations.map((Z)=>({span_id:Z.spanId,span_attribute:Z.spanAttribute}))}}}),data_object:W.toJSON()}});return await this._client.postV1projectsEvalResultsExamples(R,{results:Y,run:D}),q.length}}var B1=VR(YR(),1);class IR extends DR{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:P.map((U)=>({name:U})),custom_scorers:[]}}async _submit(R,f,q,$,D){return await this._client.postV1projectsEvalQueueExamples(R,D),console.log(`${B1.default.green("✓")} Evaluation submitted`),q.length}}class WR{_local;_hosted;constructor(R,f,q){this._local=new vR(R,f,q),this._hosted=new IR(R,f,q)}run(R){let{examples:f,scorers:q,evalRunName:$,assertTest:D=!1,timeoutSeconds:P=300}=R,U=q.filter((O)=>typeof O==="string"),E=q.filter((O)=>O instanceof QR);if(E.length>0&&U.length>0)return F.error("Running both local and hosted scorers is not supported. Please run your evaluation with either local or hosted scorers, but not both."),Promise.resolve([]);if(E.length===0&&U.length===0)return F.error("No valid local or hosted scorers provided. Please provide at least one local or hosted scorer."),Promise.resolve([]);if(E.length>0)return this._local.run(f,E,$,D,P);return this._hosted.run(f,U,$,D,P)}}class KR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}create(){return new WR(this._client,this._projectId,this._projectName)}}HR();j();HR();class PR{name;projectId;projectName;datasetKind;examples;_client;constructor(R){this.name=R.name,this.projectId=R.projectId,this.projectName=R.projectName,this.datasetKind=R.datasetKind??"example",this.examples=R.examples??[],this._client=R.client??null}async addExamples(R,f=100){if(!this._client)return;for(let q=0;q<R.length;q+=f){let $=R.slice(q,q+f);await this._client.postV1projectsDatasetsByDatasetIdentifierExamples(this.projectId,this.name,{examples:$.map((D)=>D.toJSON())})}}async addFromJson(R,f=100){let{readFile:q}=await import("fs/promises"),$=await q(R,"utf-8"),P=JSON.parse($).map((U)=>{if(typeof U!=="object"||U===null)throw Error("Each item in the JSON array must be an object");return m.create(U)});await this.addExamples(P,f)}get length(){return this.examples.length}[Symbol.iterator](){return this.examples[Symbol.iterator]()}toString(){return`Dataset(name=${this.name}, examples=${this.examples.length})`}}var nf="trace";function D1(R){if(typeof R!=="object"||R===null||Array.isArray(R))throw Error("Dataset schema must be a JSON object.");let f=R;if(f.type!=="object")throw Error('Dataset schema must declare top-level type "object".');let q=f.properties;if(typeof q!=="object"||q===null||Array.isArray(q))throw Error("Dataset schema must declare a 'properties' object.");let $=Object.entries(q).filter(([,D])=>typeof D==="object"&&D!==null&&D.type===nf).map(([D])=>D);if($.length>1)throw Error(`A dataset may declare at most one trace column; found ${$.length}: ${$.join(", ")}.`)}function gf(R){if(typeof R==="boolean")return"boolean";if(typeof R==="number")return Number.isInteger(R)?"integer":"number";if(typeof R==="string")return"string";if(Array.isArray(R))return"array";if(typeof R==="object"&&R!==null)return"object";return"string"}function P1(R){if(R.length===0)throw Error("Cannot infer a dataset schema without examples. Pass an explicit `schema` to client.datasets.create().");let f={},q=null;for(let $ of R){let D=new Set;for(let[P,U]of Object.entries($.properties)){if(U===null||U===void 0)continue;if(D.add(P),!(P in f))f[P]={type:gf(U)}}if(q===null)q=D;else if(q.size!==D.size||![...q].every((P)=>D.has(P)))throw Error("All examples must share the same set of fields (dataset schemas require all declared properties). Pass an explicit `schema` or make all examples homogeneous.")}return{type:"object",properties:f}}class uR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async get(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsDatasetsByDatasetIdentifier(f,R),$=q.dataset_kind??"example",D=(q.examples??[]).map((P)=>m.from({...P.data??{},example_id:P.example_id,created_at:P.created_at??""}));return new PR({name:R,projectId:f,projectName:this._projectName,datasetKind:$,examples:D,client:this._client})}async create(R,f={}){let q=this._expectProjectId();if(!q)return null;let{examples:$=[],overwrite:D=!1,batchSize:P=100,schema:U}=f,E;if(U!==void 0)D1(U),E=U;else{if($.length===0)throw Error("Datasets require a JSON Schema. Pass `schema` to create(), or provide `examples` to infer one.");E=P1($)}await this._client.postV1projectsDatasets(q,{name:R,examples:[],dataset_kind:"example",overwrite:D,schema:E});let O=new PR({name:R,projectId:q,projectName:this._projectName,examples:$,client:this._client});if($.length>0)await O.addExamples($,P);return O}list(){let R=this._expectProjectId();if(!R)return Promise.resolve(null);return this._client.getV1projectsDatasets(R)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Dataset operations require a valid project."),null;return this._projectId}}j();class SR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async create(R){let f=this._expectProjectId();if(!f)return null;let q={name:R.name,prompt:R.prompt,model:R.model,score_type:R.scoreType};if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;return{judgeId:(await this._client.postV1projectsJudges(f,q)).judge_id,name:R.name,prompt:R.prompt,model:R.model,scoreType:R.scoreType,description:R.description??null,judgeDescription:R.judgeDescription??null,categories:R.categories??null,minScore:R.minScore??null,maxScore:R.maxScore??null,majorVersion:0,minorVersion:0}}async update(R){let f=this._expectProjectId();if(!f)return null;let q={};if(R.prompt!==void 0)q.prompt=R.prompt;if(R.model!==void 0)q.model=R.model;if(R.scoreType!==void 0)q.score_type=R.scoreType;if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;if(R.sourceMajorVersion!==void 0)q.source_major_version=R.sourceMajorVersion;if(R.sourceMinorVersion!==void 0)q.source_minor_version=R.sourceMinorVersion;if(R.targetMajorVersion!==void 0)q.target_major_version=R.targetMajorVersion;if(R.targetMinorVersion!==void 0)q.target_minor_version=R.targetMinorVersion;let $=await this._client.patchV1projectsJudgesByJudgeId(f,R.judgeId,q);return of($)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Agent judge operations require a valid project."),null;return this._projectId}}function of(R){let f=R.judge;return{judgeId:f.id,name:f.name,prompt:f.prompt??"",model:f.model??"",scoreType:f.score_type,description:f.description??null,judgeDescription:f.judge_description??null,categories:f.categories??null,minScore:f.min_score??null,maxScore:f.max_score??null,majorVersion:f.major_version??null,minorVersion:f.minor_version??null}}j();j();tR();cR();B();function rR(R){let f=Array.isArray(R.judges)?R.judges.filter((q)=>typeof q==="object"&&q!==null):[];return{id:typeof R.id==="string"?R.id:"",name:typeof R.name==="string"?R.name:"",datasetId:typeof R.dataset_id==="string"?R.dataset_id:"",description:typeof R.description==="string"?R.description:null,createdAt:typeof R.created_at==="string"?R.created_at:null,judges:f}}function zf(R){let f=[];for(let q of R)for(let $ of q.scorers)if($.success!==null&&$.success!==void 0)f.push($.success);if(f.length===0)return null;return f.every((q)=>q)}var _=VR(YR(),1);function x(R){return typeof R==="object"&&R!==null&&!Array.isArray(R)?R:{}}function e(R){return Array.isArray(R)?R:[]}function M(R){return typeof R==="string"?R:""}function jf(R){return R?"Yes":"No"}function vq(R){let{score_type:f,bool_value:q,str_value:$,num_value:D}=R;if(f==="binary")return typeof q==="boolean"?jf(q):null;if(f==="categorical")return typeof $==="string"?$:null;if(f==="numeric")return typeof D==="number"?D:null;if(typeof q==="boolean")return jf(q);if(typeof $==="string")return $;if(typeof D==="number")return D;return null}function Iq(R){if(typeof R==="object"&&R!==null){let f=R.text;return typeof f==="string"&&f?f:null}if(typeof R==="string"){try{let f=JSON.parse(R);if(typeof f==="object"&&f!==null){let q=f.text;if(typeof q==="string")return q||null}}catch{}return R||null}return null}function xf(R){if(!R||R.length===0)return;let f=[];for(let q of R){if(!q.name&&!q.judgeId)throw Error("judgeVersions entries require a 'name' (or 'judgeId') key");let $={};if(q.judgeId!==void 0)$.judge_id=q.judgeId;if(q.name!==void 0)$.name=q.name;if(q.tag!==void 0)$.tag=q.tag;if(q.version!==void 0)$.version=q.version;if(q.majorVersion!==void 0)$.major_version=q.majorVersion;if(q.minorVersion!==void 0)$.minor_version=q.minorVersion;f.push($)}return f}function mf(R,f,q){let $=[];for(let D of R){let P=M(D.example_id),U=x(D.data),E=[];for(let O of e(D.scorers)){let Q=x(O),Y={judge_id:Q.judge_id,judge_major_version:Q.judge_major_version,judge_minor_version:Q.judge_minor_version},W=Iq(Q.reason);if(W)Y.reason=W;E.push({name:M(Q.judge_name),value:vq(Q),scoreType:typeof Q.score_type==="string"?Q.score_type:null,error:typeof Q.error==="string"?Q.error:null,success:typeof Q.success==="boolean"?Q.success:null,metadata:Y})}if(q){let O=Boolean(q({...U},E));for(let Q of E)Q.success=O}$.push({exampleId:P,data:U,scorers:E,agentOfflineTraceId:f[P]})}return $}function kf(R){if(R.status!=="completed")throw Error(`Test run ${R.testRunId} finished with status '${R.status}'`);let f=R.results.filter((q)=>q.scorers.some(($)=>$.success===!1)).map((q)=>q.exampleId);if(f.length>0||R.passed!==!0)throw Error(`Test run ${R.testRunId} failed its pass condition for ${f.length} example(s): ${JSON.stringify(f)}`)}function yf(R,f,q){console.log(),console.log(_.default.bold(_.default.cyan("Starting Offline Test"))),console.log(`${_.default.dim("Config:")} ${R}`),console.log(`${_.default.dim("Project:")} ${f}`),console.log(`${_.default.dim("Examples:")} ${q}`),console.log()}function Lf(R,f){if(console.log(),R.forEach((q,$)=>{if(q.scorers.some((P)=>P.success!==null&&P.success!==void 0)){let P=q.scorers.every((U)=>U.success===!0);console.log(P?`${_.default.green("✓")} Example ${$+1}: ${_.default.green("PASSED")}`:`${_.default.red("✗")} Example ${$+1}: ${_.default.red("FAILED")}`)}else console.log(`${_.default.cyan("•")} Example ${$+1}:`);for(let P of q.scorers){if(P.error){console.log(` ${_.default.dim(`${P.name}:`)} ${_.default.red(`error: ${P.error}`)}`);continue}let U=P.value===null?"N/A":String(P.value),E=P.success===!0?_.default.green(U):P.success===!1?_.default.red(U):U;console.log(` ${_.default.dim(`${P.name}:`)} ${E}`)}}),console.log(),f)console.log(`${_.default.dim("View full details:")} ${_.default.underline(f)}`),console.log()}var Kq=new Set(["completed","error","cancelled"]),uq=100,lq=200,pq=2000,eR=1e4;class zR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async resolveDatasetVersion(R,f){let q=await this._client.getV1projectsDatasetsByDatasetIdentifierVersions(this._projectId,R.datasetId),$=e(q.versions).filter((D)=>typeof D==="object"&&D!==null).map((D)=>D);if(f===void 0){if($.length===0)throw Error(`Dataset of test config '${R.name}' has no versions`);return $.reduce((D,P)=>Number(P.version_number??0)>Number(D.version_number??0)?P:D)}if(typeof f==="number"){let D=$.find((P)=>Number(P.version_number??0)===f);if(D)return D}else{let D=$.find((P)=>P.version_id===f);if(D)return D}throw Error(`Dataset version ${JSON.stringify(f)} does not exist for the dataset of test config '${R.name}'`)}async fetchExamples(R,f){let q=[],$,D,P=0;for(;;){if(P>=eR)throw Error(`fetchExamples exceeded ${eR} pages for dataset of test config '${R.name}'`);let U=await this._client.getV1projectsDatasetsByDatasetIdentifierPage(this._projectId,R.datasetId,String(f),String(uq),$,D);P+=1;for(let Q of e(U.entries)){let Y=x(x(Q).example),W=Y.data;if(typeof W==="string")try{W=JSON.parse(W)}catch{W={}}q.push({exampleId:M(Y.example_id),data:x(W),offlineTraceId:typeof Y.offline_trace_id==="string"?Y.offline_trace_id:null,createdAt:typeof Y.created_at==="string"?Y.created_at:null})}let E=x(U.metadata),O=x(E.nextCursor);if(!E.hasMore||Object.keys(O).length===0)break;$=M(O.created_at),D=M(O.example_id)}return q}createTestRun(R,f={}){let q={test_config_id:R.id,source:f.source??"sdk"};if(f.name)q.name=f.name;if(typeof f.datasetVersion==="number")q.dataset_version_number=f.datasetVersion;else if(typeof f.datasetVersion==="string")q.dataset_version_id=f.datasetVersion;let $=xf(f.judgeVersions);if($)q.judge_versions=$;let D=f.agentTraces??{};if(Object.keys(D).length>0)q.agent_traces=Object.entries(D).map(([P,U])=>({example_id:P,agent_offline_trace_id:U}));return this._client.postV1projectsTestRuns(this._projectId,q)}async runAgent(R,f){let q=[],$=G.getInstance().getActiveTracer();await r.create({projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl(),dataset:q,setActive:!0});let D={};try{let P=I.observe(R,{spanType:"agent"});for(let U of f){let E=q.length;try{await P(U.data)}catch(O){F.error(`Agent entrypoint raised for example ${U.exampleId}: ${String(O)}`)}for(let O of q.slice(E)){let Q=O.get("offline_trace_id");if(U.exampleId&&typeof Q==="string"){D[U.exampleId]=Q;break}}}}finally{if(await I.forceFlush(),$)$.setActive()}return D}async waitForCompletion(R,f){let q=Date.now();for(;;){if((Date.now()-q)/1000>f)throw Error(`Test run ${R} did not complete within ${f}s`);let $=await this._client.getV1projectsTestRunsByTestRunId(this._projectId,R),D=M(x($.test_run).status);if(Kq.has(D))return D;await LR(pq)}}async fetchItems(R){let f=[],q="",$,D=0;for(;;){if(D>=eR)throw Error(`fetchItems exceeded ${eR} pages for test run ${R}`);let P=await this._client.getV1projectsTestRunsByTestRunIdItems(this._projectId,R,String(lq),$);D+=1;for(let E of e(P.results))f.push(x(E));if(!q)q=M(P.ui_results_url);let U=P.next_cursor;if(!P.has_more||typeof U!=="string"||!U)break;$=U}return{items:f,uiResultsUrl:q}}async reportSuccess(R,f,q,$){let D=new Map,P=new Map,U=(Y,W,C,J)=>`${Y}\x00${W}\x00${C}\x00${J}`,E=(Y,W)=>`${Y}\x00${W}`;for(let Y of e(f.evaluation_runs)){let W=x(Y),C=W.run_id;if(typeof C!=="string"||!C)continue;D.set(U(M(W.example_id),M(W.judge_id),Number(W.judge_major_version??0),Number(W.judge_minor_version??0)),C),P.set(E(M(W.example_id),M(W.judge_name)),C)}let O=new Map;for(let Y of $)O.set(Y.exampleId,Y);let Q=[];for(let Y of q){let W=M(Y.example_id),C=O.get(W),J=C?C.scorers.map((h)=>h.success??null):[],b=e(Y.scorers);for(let h=0;h<b.length;h+=1){let Z=x(b[h]),T=D.get(U(W,M(Z.judge_id),Number(Z.judge_major_version??0),Number(Z.judge_minor_version??0)))??P.get(E(W,M(Z.judge_name)));if(!T){F.warning(`No evaluation run ref for scorer ${M(Z.judge_name)} of example ${W}; skipping its success update`);continue}Q.push({evaluation_run_id:T,success:h<J.length?J[h]:null})}}if(Q.length===0)return;await this._client.patchV1projectsTestRunsByTestRunIdSuccess(this._projectId,R,{successes:Q})}async run(R,f={}){let{agentFunction:q,judgeVersions:$,datasetVersion:D,passConditionFn:P,assertTest:U=!1,timeoutSeconds:E=600,runName:O}=f;if(U&&!P)throw Error("assertTest=true requires a passConditionFn to decide whether each row passes.");let Q=await this.resolveDatasetVersion(R,D),Y=Number(Q.version_number??0),W=await this.fetchExamples(R,Y);yf(R.name,this._projectName,W.length),F.info(`Dataset version ${Y}: ${W.length} example(s)`);let C=typeof D==="string"?D:Y,J={};if(q&&W.length>0)J=await this.runAgent(q,W);let b=await this.createTestRun(R,{datasetVersion:C,judgeVersions:$,agentTraces:J,name:O}),h=x(b.test_run),Z=M(h.id),T=M(b.ui_results_url);F.info(`Created test run ${Z} over ${W.length} examples`);let xR=await this.waitForCompletion(Z,E),{items:mR,uiResultsUrl:L}=await this.fetchItems(Z);if(L)T=L;let p=mf(mR,J,P);if(P)await this.reportSuccess(Z,b,mR,p);Lf(p,T||void 0);let a={testRunId:Z,status:xR,uiResultsUrl:T||void 0,results:p,agentOfflineTraceIds:J,passed:zf(p)};if(U)kf(a);return a}}var dq=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function T1(R){return dq.test(R)}class jR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async createConfig(R,f,q,$){let D=this._expectProjectId();if(!D)return null;let P=[];for(let O of q)if(typeof O==="string")P.push(T1(O)?{judge_id:O}:{name:O});else if(O.judgeId)P.push({judge_id:O.judgeId});else if(O.name)P.push({name:O.name});else throw Error("judges entries must be judge names or objects with a 'judgeId' or 'name'");let U={name:R,judges:P};if($!==void 0)U.description=$;if(T1(f))U.dataset_id=f;else U.dataset_name=f;let E=await this._client.postV1projectsTestConfigs(D,U);return F.info(`Created test config ${R}`),rR(E.test_config??{})}async getConfig(R){let f=this._expectProjectId();if(!f)return null;if(T1(R)){let D=await this._client.getV1projectsTestConfigsByTestConfigId(f,R);return rR(D.test_config??{})}let $=(await this.listConfigs()??[]).find((D)=>D.name===R);if(!$)return F.error(`Test config '${R}' not found`),null;return $}async listConfigs(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsTestConfigs(f,R);return(Array.isArray(q.test_configs)?q.test_configs:[]).map((D)=>rR(D))}async deleteConfig(R){let f=this._expectProjectId();if(!f)return!1;return await this._client.deleteV1projectsTestConfigsByTestConfigId(f,R),F.info(`Deleted test config ${R}`),!0}async run(R,f={}){let q=this._expectProjectId();if(!q)return null;let $=typeof R==="string"?await this.getConfig(R):R;if(!$||!$.id)return null;return new zR(this._client,q,this._projectName).run($,f)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Offline-test operations require a valid project."),null;return this._projectId}}class z1{_client;_projectName;_projectId;constructor(R,f,q){this._client=R,this._projectName=f,this._projectId=q}static async create(R){let f=R.apiKey??RR,q=R.organizationId??fR,$=R.apiUrl??qR;if(!f)throw Error("API key is required");if(!q)throw Error("Organization ID is required");if(!$)throw Error("API URL is required");if(!R.projectName)throw Error("Project name is required");let D=new s($,f,q),P=null;try{P=await $R(D,R.projectName)}catch{F.warning(`Project '${R.projectName}' not found. Some operations requiring project_id will be skipped.`)}return new z1(D,R.projectName,P)}async offlineTracer(R){let{OfflineTracer:f}=await Promise.resolve().then(() => (cR(),Tf));return f.create({...R,projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl()})}get datasets(){return new uR(this._client,this._projectId,this._projectName)}get evaluation(){return new KR(this._client,this._projectId,this._projectName)}get agentJudges(){return new SR(this._client,this._projectId,this._projectName)}get offlineTests(){return new jR(this._client,this._projectId,this._projectName)}}K();bR();oR();B();_R();_1();M1();w1();iR();JR();J1();tR();cR();Y1();HR();export{F1 as wrapOpenAI,Q1 as wrap,D1 as validateDatasetSchema,G1 as propagation,P1 as inferSchemaFromExamples,Z1 as baggage,I as Tracer,r as OfflineTracer,jR as OfflineTestsFactory,zR as OfflineTestRunner,TR as OfflineJudgmentSpanProcessor,MR as NoOpSpanProcessor,c as NoOpSpanExporter,G as JudgmentTracerProvider,v as JudgmentSpanExporter,wR as JudgmentBaggageSpanProcessor,t as JudgmentBaggagePropagator,z1 as Judgeval,QR as Judge,m as Example,WR as Evaluation,PR as Dataset,V as BaseTracer,SR as AgentJudgeFactory,b1 as ALLOW_ALL_BAGGAGE_KEYS};
5
-
6
- //# debugId=B736D4C79A323AD864756E2164756E21
7
- //# sourceMappingURL=index.mjs.map