judgeval 1.1.0 → 1.2.1

package/dist/node/index.cjs

@@ -0,0 +1,7 @@
+var $q=Object.create;var{getPrototypeOf:Dq,defineProperty:XR,getOwnPropertyNames:n1,getOwnPropertyDescriptor:Pq}=Object,g1=Object.prototype.hasOwnProperty;var JR=(R,f,q)=>{q=R!=null?$q(Dq(R)):{};let $=f||!R||!R.__esModule?XR(q,"default",{value:R,enumerable:!0}):q;for(let D of n1(R))if(!g1.call($,D))XR($,D,{get:()=>R[D],enumerable:!0});return $},s1=new WeakMap,Uq=(R)=>{var f=s1.get(R),q;if(f)return f;if(f=XR({},"__esModule",{value:!0}),R&&typeof R==="object"||typeof R==="function")n1(R).map(($)=>!g1.call(f,$)&&XR(f,$,{get:()=>R[$],enumerable:!(q=Pq(R,$))||q.enumerable}));return s1.set(R,f),f},Eq=(R,f)=>()=>(f||R((f={exports:{}}).exports,f),f.exports);var BR=(R,f)=>{for(var q in f)XR(R,q,{get:f[q],enumerable:!0,configurable:!0,set:($)=>f[q]=()=>$})};var H=(R,f)=>()=>(R&&(f=R(R=0)),f);function vR(R,f){let q=process.env[R];if(!q)return f??null;return q}var qR,$R,DR,i1;var GR=H(()=>{qR=vR("JUDGMENT_API_KEY"),$R=vR("JUDGMENT_ORG_ID"),DR=vR("JUDGMENT_API_URL","https://api.judgmentlabs.ai"),i1=vR("JUDGMENT_LOG_LEVEL","warn")});class c{baseUrl;apiKey;organizationId;constructor(R,f,q){this.baseUrl=R,this.apiKey=f,this.organizationId=q}getBaseUrl(){return this.baseUrl}getApiKey(){return this.apiKey}getOrganizationId(){return this.organizationId}async request(R,f,q){let $=await fetch(f,{method:R,headers:{"Content-Type":"application/json",Authorization:`Bearer ${this.apiKey}`,"X-Organization-Id":this.organizationId},body:q!==void 0?JSON.stringify(q):void 0});if(!$.ok){let D=await $.text();throw Error(`HTTP ${$.status}: ${D}`)}return $.json()}async postOtelV1traces(){let R=this.baseUrl+"/otel/v1/traces";return this.request("POST",R,{})}async postOtelV1offlineTraces(){let R=this.baseUrl+"/otel/v1/offline-traces";return this.request("POST",R,{})}async postV1projectsResolve(R){let f=this.baseUrl+"/v1/projects/resolve/";return this.request("POST",f,R)}async postV1projects(R){let f=this.baseUrl+"/v1/projects";return this.request("POST",f,R)}async deleteV1projects(R){let f=this.baseUrl+`/v1/projects/${R}`;return this.request("DELETE",f,{})}async postV1projectsDatasets(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("POST",q,f)}async getV1projectsDatasets(R){let f=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("GET",f,void 0)}async getV1projectsDatasetsByDatasetIdentifierVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsDatasetsByDatasetIdentifierPage(R,f,q,$,D,P){let U=new URLSearchParams;if(q!==void 0)U.set("version",q);if($!==void 0)U.set("limit",$);if(D!==void 0)U.set("cursor_created_at",D);if(P!==void 0)U.set("cursor_example_id",P);let O=this.baseUrl+`/v1/projects/${R}/datasets/${f}/page`+(U.toString()?"?"+U.toString():"");return this.request("GET",O,void 0)}async postV1projectsDatasetsByDatasetIdentifierExamples(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/datasets/${f}/examples`;return this.request("POST",$,q)}async deleteV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("DELETE",q,{})}async getV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("GET",q,void 0)}async getV1projectsTestConfigs(R,f){let q=new URLSearchParams;if(f!==void 0)q.set("dataset_id",f);let $=this.baseUrl+`/v1/projects/${R}/test-configs`+(q.toString()?"?"+q.toString():"");return this.request("GET",$,void 0)}async postV1projectsTestConfigs(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs`;return this.request("POST",q,f)}async getV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestConfigsByTestConfigId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("PATCH",$,q)}async deleteV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("DELETE",q,{})}async getV1projectsTestRuns(R,f,q,$,D){let P=new URLSearchParams;if(f!==void 0)P.set("test_config_id",f);if(q!==void 0)P.set("dataset_id",q);if($!==void 0)P.set("status",$);if(D!==void 0)P.set("limit",D);let U=this.baseUrl+`/v1/projects/${R}/test-runs`+(P.toString()?"?"+P.toString():"");return this.request("GET",U,void 0)}async postV1projectsTestRuns(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs`;return this.request("POST",q,f)}async getV1projectsTestRunsByTestRunId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestRunsByTestRunIdStatus(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/status`;return this.request("PATCH",$,q)}async patchV1projectsTestRunsByTestRunIdSuccess(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/success`;return this.request("PATCH",$,q)}async getV1projectsTestRunsByTestRunIdItems(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/items`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdResults(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/results`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdGraph(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/graph`;return this.request("GET",q,void 0)}async getV1projectsExperimentsByRunId(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/experiments/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsEvalResults(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results`;return this.request("POST",q,f)}async postV1projectsEvalResultsExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/traces`;return this.request("POST",q,f)}async postV1projectsEvalQueueExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/examples`;return this.request("POST",q,f)}async postV1projectsEvalQueueTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/traces`;return this.request("POST",q,f)}async getV1projectsPromptsByName(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("commit_id",q);if($!==void 0)D.set("tag",$);let P=this.baseUrl+`/v1/projects/${R}/prompts/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsPrompts(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts`;return this.request("POST",q,f)}async postV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("POST",$,q)}async deleteV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("DELETE",$,q)}async getV1projectsPromptsByNameVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsScorers(R,f,q){let $=new URLSearchParams;if(f!==void 0)$.set("names",f);if(q!==void 0)$.set("is_trace",q);let D=this.baseUrl+`/v1/projects/${R}/scorers`+($.toString()?"?"+$.toString():"");return this.request("GET",D,void 0)}async getV1projectsScorersByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsScorersCustom(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom`;return this.request("POST",f,{})}async postV1projectsScorersCustomBundle(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom/bundle`;return this.request("POST",f,{})}async getV1projectsScorersCustomByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/custom/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsTracesByTraceIdTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/traces/${f}/tags`;return this.request("POST",$,q)}async getV1e2eFetchTraceByProjectNameByTraceId(R,f){let q=this.baseUrl+`/v1/e2e_fetch_trace/${R}/${f}`;return this.request("GET",q,void 0)}async getV1e2eTracesPerProject(R){let f=this.baseUrl+`/v1/e2e_traces_per_project/${R}`;return this.request("GET",f,void 0)}async postV1e2eFetchSpanScore(R){let f=this.baseUrl+"/v1/e2e_fetch_span_score/";return this.request("POST",f,R)}async postV1projectsJudges(R,f){let q=this.baseUrl+`/v1/projects/${R}/judges`;return this.request("POST",q,f)}async patchV1projectsJudgesByJudgeId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/judges/${f}`;return this.request("PATCH",$,q)}}var IR=()=>{};var F;var T=H(()=>{GR();F=class F{static RESET="\x1B[0m";static RED="\x1B[31m";static YELLOW="\x1B[33m";static GRAY="\x1B[90m";static Level={DEBUG:0,INFO:1,WARNING:2,ERROR:3,CRITICAL:4};static initialized=!1;static levelSetManually=!1;static currentLevel=F.Level.WARNING;static useColor=!0;static initialize(){if(!F.initialized){let R=process.env.JUDGMENT_NO_COLOR;if(F.useColor=!R&&process.stdout.isTTY,!F.levelSetManually){let f=i1.toLowerCase();if(f){let q={debug:F.Level.DEBUG,info:F.Level.INFO,warning:F.Level.WARNING,warn:F.Level.WARNING,error:F.Level.ERROR,critical:F.Level.CRITICAL};F.currentLevel=q[f]??F.Level.WARNING}}F.initialized=!0}}static setLevel(R){F.currentLevel=R,F.levelSetManually=!0}static setUseColor(R){F.useColor=R}static log(R,f){if(F.initialize(),R<F.currentLevel)return;let q=new Date().toISOString().replace("T"," ").substring(0,19),$=Object.keys(F.Level).find((U)=>F.Level[U]===R)??"UNKNOWN",D=`${q} - judgeval - ${$} - ${f}`;if(F.useColor)D=`${R===F.Level.DEBUG||R===F.Level.INFO?F.GRAY:R===F.Level.WARNING?F.YELLOW:F.RED}${D}${F.RESET}`;(R>=F.Level.ERROR?process.stderr:process.stdout).write(D+`
+`)}static debug(R){F.log(F.Level.DEBUG,R)}static info(R){F.log(F.Level.INFO,R)}static warning(R){F.log(F.Level.WARNING,R)}static warn(R){F.log(F.Level.WARNING,R)}static error(R){F.log(F.Level.ERROR,R)}static critical(R){F.log(F.Level.CRITICAL,R)}}});function KR(R){return new Promise((f)=>setTimeout(f,R))}async function o1(R,f={}){let{maxRetries:q=3,backoff:$=()=>1000,onRetry:D}=f;for(let P=1;P<=q;P++)try{return await R()}catch(U){if(P===q)throw U;D?.(P,U),await KR($(P))}throw Error("retry: exhausted all attempts")}var t1=()=>{};async function PR(R,f){let q=`org:${R.getOrganizationId()}:project:${f}`,$=c1.get(q);if($)return $;let D=J1.get(q);if(D)return D;let P=(async()=>{F.info(`Resolving project ID for project: ${f}`);let U=await o1(async()=>{let E=(await R.postV1projectsResolve({project_name:f})).project_id;if(!E)throw Error(`Project ID not found for project: ${f}`);return E},{maxRetries:3,backoff:(O)=>O*1000,onRetry:(O,E)=>{F.warning(`Failed to resolve project ID for '${f}' (attempt ${O}): ${String(E)}`)}});return F.info(`Resolved project ID: ${U}`),c1.set(q,U),U})();J1.set(q,P);try{return await P}finally{J1.delete(q)}}var c1,J1;var uR=H(()=>{T();t1();c1=new Map,J1=new Map});var hR=Eq((P$,G1)=>{var pR=process||{},r1=pR.argv||[],dR=pR.env||{},Oq=!(!!dR.NO_COLOR||r1.includes("--no-color"))&&(!!dR.FORCE_COLOR||r1.includes("--color")||pR.platform==="win32"||(pR.stdout||{}).isTTY&&dR.TERM!=="dumb"||!!dR.CI),Vq=(R,f,q=R)=>($)=>{let D=""+$,P=D.indexOf(f,R.length);return~P?R+Fq(D,f,q,P)+f:R+D+f},Fq=(R,f,q,$)=>{let D="",P=0;do D+=R.substring(P,$)+q,P=$+f.length,$=R.indexOf(f,P);while(~$);return D+R.substring(P)},e1=(R=Oq)=>{let f=R?Vq:()=>String;return{isColorSupported:R,reset:f("\x1B[0m","\x1B[0m"),bold:f("\x1B[1m","\x1B[22m","\x1B[22m\x1B[1m"),dim:f("\x1B[2m","\x1B[22m","\x1B[22m\x1B[2m"),italic:f("\x1B[3m","\x1B[23m"),underline:f("\x1B[4m","\x1B[24m"),inverse:f("\x1B[7m","\x1B[27m"),hidden:f("\x1B[8m","\x1B[28m"),strikethrough:f("\x1B[9m","\x1B[29m"),black:f("\x1B[30m","\x1B[39m"),red:f("\x1B[31m","\x1B[39m"),green:f("\x1B[32m","\x1B[39m"),yellow:f("\x1B[33m","\x1B[39m"),blue:f("\x1B[34m","\x1B[39m"),magenta:f("\x1B[35m","\x1B[39m"),cyan:f("\x1B[36m","\x1B[39m"),white:f("\x1B[37m","\x1B[39m"),gray:f("\x1B[90m","\x1B[39m"),bgBlack:f("\x1B[40m","\x1B[49m"),bgRed:f("\x1B[41m","\x1B[49m"),bgGreen:f("\x1B[42m","\x1B[49m"),bgYellow:f("\x1B[43m","\x1B[49m"),bgBlue:f("\x1B[44m","\x1B[49m"),bgMagenta:f("\x1B[45m","\x1B[49m"),bgCyan:f("\x1B[46m","\x1B[49m"),bgWhite:f("\x1B[47m","\x1B[49m"),blackBright:f("\x1B[90m","\x1B[39m"),redBright:f("\x1B[91m","\x1B[39m"),greenBright:f("\x1B[92m","\x1B[39m"),yellowBright:f("\x1B[93m","\x1B[39m"),blueBright:f("\x1B[94m","\x1B[39m"),magentaBright:f("\x1B[95m","\x1B[39m"),cyanBright:f("\x1B[96m","\x1B[39m"),whiteBright:f("\x1B[97m","\x1B[39m"),bgBlackBright:f("\x1B[100m","\x1B[49m"),bgRedBright:f("\x1B[101m","\x1B[49m"),bgGreenBright:f("\x1B[102m","\x1B[49m"),bgYellowBright:f("\x1B[103m","\x1B[49m"),bgBlueBright:f("\x1B[104m","\x1B[49m"),bgMagentaBright:f("\x1B[105m","\x1B[49m"),bgCyanBright:f("\x1B[106m","\x1B[49m"),bgWhiteBright:f("\x1B[107m","\x1B[49m")}};G1.exports=e1();G1.exports.createColors=e1});var x;var wR=H(()=>{x=class x{exampleId;createdAt;name;_properties;constructor(R,f,q,$){this.exampleId=R,this.createdAt=f,this.name=q,this._properties=$}static create(R={}){return new x(crypto.randomUUID(),new Date().toISOString(),null,{...R})}static META_KEYS=new Set(["example_id","created_at","name","trace_id","span_id","offline_trace_id","agent_offline_trace_id"]);static from(R){let f={};for(let q of Object.keys(R))if(!x.META_KEYS.has(q))f[q]=R[q];return new x(R.example_id??"",R.created_at??"",R.name??null,f)}get(R){return this._properties[R]}has(R){return R in this._properties}get properties(){return{...this._properties}}toJSON(){let R={example_id:this.exampleId,created_at:this.createdAt,name:this.name};for(let[f,q]of Object.entries(this._properties))R[f]=q;return R}}});function Sq(){let R=new WeakSet;return function(f,q){if(typeof q==="bigint")return q.toString();if(typeof q==="object"&&q!==null){if(R.has(q))return"[Circular]";R.add(q)}return q}}function w(R){try{let f=JSON.stringify(R);if(typeof f==="string")return f;return String(f)}catch{try{let f=JSON.stringify(R,Sq());return typeof f==="string"?f:String(R)}catch(f){return F.error(`safeStringify failed: ${f}`),String(R)}}}function e(R,f){if(typeof R==="string"||typeof R==="number"||typeof R==="boolean")return R;return f(R)}var I=H(()=>{T()});var ff="1.2.1";var Rf=()=>{};var tR;var w1=H(()=>{Rf();tR=ff});var FR=()=>{};function S(R,f,q){try{return f()}catch($){let D=$ instanceof Error&&$.stack?`
+${$.stack}`:"";return F.error(`[Caught] An exception was raised in ${R}: ${String($)}${D}`),q}}var k=H(()=>{T()});var qf=H(()=>{k()});function K(R,f={}){let{pre:q,post:$,error:D,finally:P}=f;return async function(...O){let E=q?S("immutableWrapAsync.pre",()=>q(...O)):void 0,Q;try{let Y=await R.apply(this,O);if($)Q=S("immutableWrapAsync.post",()=>$(E,Y,O));return Y}catch(Y){if(D)Q=S("immutableWrapAsync.error",()=>D(E,Y,O));throw Y}finally{if(P)S("immutableWrapAsync.finally",()=>{P(Q)})}}}var $f=H(()=>{k()});var Df=H(()=>{k()});function _1(R,f={}){let{pre:q,yield:$,post:D,error:P,finally:U}=f;return async function*(...E){let Q=q?S("immutableWrapAsyncIterator.pre",()=>q(...E)):void 0,Y;try{for await(let W of R(...E)){if($)S("immutableWrapAsyncIterator.yield",()=>{$(Q,W)});yield W}if(D)Y=S("immutableWrapAsyncIterator.post",()=>D(Q))}catch(W){if(P)Y=S("immutableWrapAsyncIterator.error",()=>P(Q,W));throw W}finally{if(U)S("immutableWrapAsyncIterator.finally",()=>{U(Y)})}}}var b1=H(()=>{k()});function a(R,f){let q=R[Symbol.asyncIterator].bind(R),$=_1(()=>({[Symbol.asyncIterator]:q}),{yield:(D,P)=>{f.onYield(P)},post:()=>{f.onDone()},error:(D,P)=>{f.onError(P)},finally:()=>{f.onFinally()}});R[Symbol.asyncIterator]=()=>$()}var Pf=H(()=>{b1()});var _R=H(()=>{qf();$f();Df();b1();Pf()});function bR(R,f){S("recordChatUsage",()=>{let q=f.prompt_tokens_details?.cached_tokens??0,$=f.prompt_tokens+f.completion_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.prompt_tokens-q:f.prompt_tokens,output_tokens:f.completion_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}var M1=H(()=>{FR();p();k();I()});function Uf(R){R.chat.completions.parse=K(R.chat.completions.parse.bind(R.chat.completions),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),q},post:(f,q)=>{if(!f)return;if(V.setOutput(w(q),f),q.usage)bR(f,q.usage);return V.recordLLMMetadata({model:q.model},f),f},error:(f,q)=>{if(f)V.setError(q,f);return f},finally:(f)=>{f?.end()}})}var Ef=H(()=>{p();I();_R();M1()});function Of(R){R.chat.completions.create=K(R.chat.completions.create.bind(R.chat.completions),{pre:(f)=>{if(f.stream)f.stream_options??={include_usage:!0};let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O="";return a(U,{onYield(E){if(typeof E.choices[0]?.delta.content==="string")O+=E.choices[0].delta.content;if(E.usage)bR(D,E.usage)},onDone(){V.setOutput(O,D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)bR(D,P.usage);return V.recordLLMMetadata({model:P.model},D),f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Vf=H(()=>{p();I();_R();M1()});function Ff(R,f){S("images.recordUsage",()=>{let q="input_tokens_details"in f?f.input_tokens_details:void 0,$=q?.image_tokens??0;if(V.recordLLMMetadata({non_cached_input_tokens:q?.text_tokens??0,output_tokens:f.output_tokens||void 0},R),$)V.setAttribute("judgment.usage.non_cached_input_image_tokens",$,R);if(f.output_tokens)V.setAttribute("judgment.usage.output_image_tokens",f.output_tokens,R);V.setAttribute("judgment.usage.metadata",w(f),R)})}function Qf(R){R.images.generate=K(R.images.generate.bind(R.images),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O;return a(U,{onYield(E){if(Zq.has(E.type))O=E,Ff(D,O.usage)},onDone(){V.setOutput(w(O??{}),D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)Ff(D,P.usage);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Zq;var Yf=H(()=>{FR();p();k();I();_R();Zq=new Set(["image_generation.completed","image_edit.completed"])});function Wf(R,f){S("responses.recordUsage",()=>{let q=f.input_tokens_details.cached_tokens,$=f.input_tokens+f.output_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.input_tokens-q:f.input_tokens,output_tokens:f.output_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}function Hf(R){R.responses.create=K(R.responses.create.bind(R.responses),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O="";return a(U,{onYield(E){if(E.type==="response.output_text.delta")O+=E.delta;if(E.type==="response.completed"){let Q=E.response;if(Q.usage)Wf(D,Q.usage);V.recordLLMMetadata({model:Q.model},D)}},onDone(){V.setOutput(O,D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)Wf(D,P.usage);if(typeof P.model==="string")V.recordLLMMetadata({model:P.model},D);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Sf=H(()=>{FR();p();k();I();_R()});function rR(R){return S("wrapOpenAI",()=>{Of(R),Uf(R),Hf(R),Qf(R)}),R}var Cf=H(()=>{k();Ef();Vf();Yf();Sf()});function eR(R){return rR(R)}var T1=H(()=>{Cf()});function _q(R){return Function.prototype.toString.call(R)}function bq(R){let f=_q(R).replace(wq,"");return f.match(Xq)||f.match(Jq)}function Zf(R){let f=bq(R);if(!f||!f[1])return[];return f[1].split(Gq).map((q)=>{return q.replace(hq,(D,P,U)=>U).trim()}).filter((q)=>q.length>0)}var Xq,Jq,Gq,hq,wq;var Xf=H(()=>{Xq=/^([^(]+?)=>/,Jq=/^[^(]*\(\s*([^)]*)\)/m,Gq=/,/,hq=/^\s*(_?)(\S+?)\1\s*$/,wq=/((\/\/.*$)|(\/\*[\s\S]*?\*\/))/gm});class Jf{startSpan(){return aR.trace.wrapSpanContext(aR.INVALID_SPAN_CONTEXT)}startActiveSpan(R,...f){return(f.length===1?f[0]:f.length===2?f[1]:f[2])(this.startSpan())}}var aR,R1;var z1=H(()=>{aR=require("@opentelemetry/api");R1=new Jf});function hf(R){Gf=R}function u(){return Gf??Mq}var MR,Mq,Gf=null;var QR=H(()=>{T();z1();MR=require("@opentelemetry/api"),Mq={register(){},deregister(){},setActive(){return!1},getActiveTracer(){return null},getCurrentContext(){return MR.ROOT_CONTEXT},setSpan(R,f){return MR.trace.setSpan(R,f)},wrapSpanContext(R){return MR.trace.wrapSpanContext(R)},getCurrentSpan(){return},getTracer(){return F.debug("No active tracer provider, returning NoOpTracer"),R1},addInstrumentation(){F.warning("No active tracer provider. Instrumentation was not registered.")},useSpan(R,f,q,$,D){return D()},attachContext(){},withContext(R,f){return f()},forceFlush(){return Promise.resolve()},shutdown(){return Promise.resolve()}}});var wf="=",f1=";",TR=",",q1="baggage",_f=180,bf=4096,Mf=8192;function zf(R){return R.reduce((f,q)=>{let $=`${f}${f!==""?TR:""}${q}`;return $.length>Mf?f:$},"")}function jf(R){return R.getAllEntries().map(([f,q])=>{let $=`${encodeURIComponent(f)}=${encodeURIComponent(q.value)}`;if(q.metadata!==void 0)$+=f1+q.metadata.toString();return $})}function xf(R){let f=R.split(f1);if(f.length<=0)return;let q=f.shift();if(!q)return;let $=q.indexOf(wf);if($<=0)return;let D=decodeURIComponent(q.substring(0,$).trim()),P=decodeURIComponent(q.substring($+1).trim()),U;if(f.length>0)U=Tf.baggageEntryMetadataFromString(f.join(f1));return{key:D,value:P,metadata:U}}var Tf;var mf=H(()=>{Tf=require("@opentelemetry/api")});class l{inject(R,f,q){let $=s(R);if(!$||kf.isTracingSuppressed(R))return;let D=jf($).filter((U)=>U.length<=bf).slice(0,_f),P=zf(D);if(P.length>0)q.set(f,q1,P)}extract(R,f,q){let $=q.get(f,q1),D=Array.isArray($)?$.join(TR):$;if(!D)return R;let P={};if(D.length===0)return R;if(D.split(TR).forEach((O)=>{let E=xf(O);if(E){let Q={value:E.value};if(E.metadata)Q.metadata=E.metadata;P[E.key]=Q}}),Object.entries(P).length===0)return R;return jR(R,zR(P))}fields(){return[q1]}}var kf;var $1=H(()=>{xR();mf();kf=require("@opentelemetry/core")});var D1={};BR(D1,{setBaggage:()=>jR,getBaggage:()=>s,getActiveBaggage:()=>Tq,deleteBaggage:()=>zq,createBaggage:()=>zR,baggageEntryMetadataFromString:()=>Lf.baggageEntryMetadataFromString,JudgmentBaggagePropagator:()=>l});function s(R){return R.getValue(j1)}function Tq(){return s(u().getCurrentContext())}function jR(R,f){return R.setValue(j1,f)}function zq(R){return R.deleteValue(j1)}var mR,Lf,zR,j1;var xR=H(()=>{QR();$1();mR=require("@opentelemetry/api"),Lf=require("@opentelemetry/api"),zR=mR.propagation.createBaggage.bind(mR.propagation),j1=mR.createContextKey("baggage")});var E1={};BR(E1,{setGlobalTextmap:()=>jq,inject:()=>xq,getGlobalTextmap:()=>x1,extract:()=>m1});function x1(){return Nf}function jq(R){Nf=R}function yf(R){if(R!==void 0)return R;return u().getCurrentContext()}function xq(R,f,q=P1.defaultTextMapSetter){S("propagation.inject",()=>{x1().inject(yf(f),R,q)})}function m1(R,f,q=P1.defaultTextMapGetter){let $=yf(f);return S("propagation.extract",()=>x1().extract($,R,q),$)}var P1,U1,Nf;var k1=H(()=>{k();$1();QR();P1=require("@opentelemetry/api"),U1=require("@opentelemetry/core"),Nf=new U1.CompositePropagator({propagators:[new U1.W3CTraceContextPropagator,new l]})});class V{projectName;projectId;apiKey;organizationId;apiUrl;environment;serializer;_tracerProvider;_client;_enableMonitoring;supportsLiveInstrumentation=!0;constructor(R,f,q,$,D,P,U,O,E,Q){this.projectName=R,this.projectId=f,this.apiKey=q,this.organizationId=$,this.apiUrl=D,this.environment=P,this.serializer=U,this._tracerProvider=O,this._client=E,this._enableMonitoring=Q}setActive(){return u().setActive(this)}static _getProxyProvider(){return u()}static _getSerializer(){return V._getProxyProvider().getActiveTracer()?.serializer??w}static _getCurrentTraceAndSpanId(){let f=V._getProxyProvider().getCurrentSpan();if(!f?.isRecording())return null;let q=f.spanContext();if(!q.traceId||!(q.traceFlags&1))return null;return[q.traceId,q.spanId]}static _emitPartial(){S("BaseTracer._emitPartial",()=>{let R=V._getProxyProvider().getActiveTracer();if(!R||!R.supportsLiveInstrumentation)return;R.getSpanProcessor().emitPartial()})}static getCurrentSpan(){return V._getProxyProvider().getCurrentSpan()}static async forceFlush(){await V._getProxyProvider().forceFlush()}static async shutdown(){await V._getProxyProvider().shutdown()}static registerOTELInstrumentation(R){S("BaseTracer.registerOTELInstrumentation",()=>{V._getProxyProvider().addInstrumentation(R)})}static wrap(R){return eR(R)}static getOTELTracer(){return V._getProxyProvider().getTracer(Af)}static startSpan(R,f){let q=V.getOTELTracer().startSpan(R,{attributes:f});return V._emitPartial(),q}static startActiveSpan(R,f){let{name:q,attributes:$}=R;return V.getOTELTracer().startActiveSpan(q,{attributes:$},(D)=>{V._emitPartial();try{let P=f(D);if(P instanceof Promise)return P.finally(()=>{D.end()});return D.end(),P}catch(P){throw D.end(),P}})}static span(R,f){return V.startActiveSpan({name:R},(q)=>{try{let $=f(q);if($ instanceof Promise)return $.catch((D)=>{throw q.setStatus({code:d.SpanStatusCode.ERROR,message:String(D)}),q.recordException(D),D});return $}catch($){throw q.setStatus({code:d.SpanStatusCode.ERROR,message:String($)}),q.recordException($),$}})}static with(R,f){return V.span(R,f)}static continueTrace(R,f){let q=V._getProxyProvider(),$=m1(R);return q.withContext($,()=>f($))}static observe(R,f){let q;if(typeof R==="function")q=R;else f=R;let{spanType:$="span",spanName:D,recordInput:P=!0,recordOutput:U=!0,fork:O=!1}=f??{},E=V._getProxyProvider(),Q=(Y)=>{let W=D??Y.name;return function(...C){let G=E.getTracer(Af);if(O&&E.getActiveTracer()!==null&&E.getCurrentSpan()?.isRecording()===!0){let X=V._getSerializer(),Z=G.startSpan(W),M=Z.spanContext();if($)Z.setAttribute("judgment.span_kind",$);let yR={"judgment.link.source_trace_id":M.traceId,"judgment.link.source_span_id":M.spanId};if($)yR["judgment.span_kind"]=$;let AR=E.setSpan(E.getCurrentContext(),E.wrapSpanContext(d.INVALID_SPAN_CONTEXT)),v=G.startSpan(W,{attributes:yR},AR),o=v.spanContext();Z.setAttribute("judgment.link.target_trace_id",o.traceId),Z.setAttribute("judgment.link.target_span_id",o.spanId);let fR=()=>{v.end(),Z.end()},p1=(z)=>{for(let N of[v,Z])N.recordException(z),N.setStatus({code:d.SpanStatusCode.ERROR,message:String(z)})},l1=(z)=>{let N=e(z,X);v.setAttribute("judgment.output",N),Z.setAttribute("judgment.output",N)};if(P){let z=e(Bf(Y,C),X);v.setAttribute("judgment.input",z),Z.setAttribute("judgment.input",z)}return V._emitPartial(),E.useSpan(v,!1,!1,!1,()=>{try{let z=Y.call(this,...C);if(z instanceof Promise)return z.then((N)=>{if(U)l1(N);return N}).catch((N)=>{throw p1(N),N}).finally(fR);if(U)l1(z);return fR(),z}catch(z){throw p1(z),fR(),z}})}return G.startActiveSpan(W,(X)=>{if($)X.setAttribute("judgment.span_kind",$);try{if(P)X.setAttribute("judgment.input",e(Bf(Y,C),V._getSerializer()));V._emitPartial();let Z=Y.call(this,...C);if(Z instanceof Promise)return Z.then((M)=>{if(U)X.setAttribute("judgment.output",e(M,V._getSerializer()));return M}).catch((M)=>{throw X.recordException(M),X.setStatus({code:d.SpanStatusCode.ERROR,message:String(M)}),M}).finally(()=>{X.end()});if(U)X.setAttribute("judgment.output",e(Z,V._getSerializer()));return X.end(),Z}catch(Z){throw X.recordException(Z),X.setStatus({code:d.SpanStatusCode.ERROR,message:String(Z)}),X.end(),Z}})}};if(!q)return Q;return Q(q)}static _resolveSpan(R){if(R)return R;return V._getProxyProvider().getCurrentSpan()}static setSpanKind(R,f){S("BaseTracer.setSpanKind",()=>{if(!R)return;let q=V._resolveSpan(f);if(q?.isRecording())q.setAttribute("judgment.span_kind",R)})}static setLLMSpan(){V.setSpanKind("llm")}static setToolSpan(){V.setSpanKind("tool")}static setGeneralSpan(){V.setSpanKind("span")}static setAttribute(R,f,q){S("BaseTracer.setAttribute",()=>{let $=V._resolveSpan(q);if(!$?.isRecording())return;if(!R||f==null)return;$.setAttribute(R,e(f,V._getSerializer()))})}static setAttributes(R,f){for(let[q,$]of Object.entries(R))if(f)V.setAttribute(q,$,f);else V.setAttribute(q,$)}static setInput(R,f){if(f)V.setAttribute("judgment.input",R,f);else V.setAttribute("judgment.input",R)}static setOutput(R,f){if(f)V.setAttribute("judgment.output",R,f);else V.setAttribute("judgment.output",R)}static setError(R,f){S("BaseTracer.setError",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;q.recordException(R),q.setStatus({code:d.SpanStatusCode.ERROR,message:String(R)})})}static recordLLMMetadata(R,f){S("BaseTracer.recordLLMMetadata",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;if(typeof R.model==="string")q.setAttribute("judgment.llm.model",R.model);if(typeof R.provider==="string")q.setAttribute("judgment.llm.provider",R.provider);if(typeof R.non_cached_input_tokens==="number")q.setAttribute("judgment.usage.non_cached_input_tokens",R.non_cached_input_tokens);if(typeof R.output_tokens==="number")q.setAttribute("judgment.usage.output_tokens",R.output_tokens);if(typeof R.cache_read_input_tokens==="number")q.setAttribute("judgment.usage.cache_read_input_tokens",R.cache_read_input_tokens);if(typeof R.cache_creation_input_tokens==="number")q.setAttribute("judgment.usage.cache_creation_input_tokens",R.cache_creation_input_tokens);if(typeof R.total_cost_usd==="number")q.setAttribute("judgment.usage.total_cost_usd",R.total_cost_usd)})}static _setPropagatingBaggageKey(R,f){S("BaseTracer._setPropagatingBaggageKey",()=>{let q=V._getProxyProvider(),$=q.getCurrentSpan();if(!$?.isRecording())return;$.setAttribute(R,f);let D=q.getCurrentContext(),P=(s(D)??zR()).setEntry(R,{value:f});q.attachContext(jR(D,P))})}static setCustomerId(R){V._setPropagatingBaggageKey("judgment.customer_id",R)}static setCustomerUserId(R){V._setPropagatingBaggageKey("judgment.customer_user_id",R)}static setSessionId(R){V._setPropagatingBaggageKey("judgment.session_id",R)}static setPropagatingAttribute(R,f){if(R.startsWith("judgment.")){F.error(`setPropagatingAttribute: key "${R}" uses the reserved "judgment." prefix; ignoring`);return}V._setPropagatingBaggageKey(R,f)}static tag(R){S("BaseTracer.tag",()=>{if(!R||Array.isArray(R)&&R.length===0)return;let q=V._getProxyProvider().getActiveTracer();if(!q?.projectId||!q._client)return;if(!q.supportsLiveInstrumentation)return;let $=V._getCurrentTraceAndSpanId();if(!$)return;let[D]=$,P=Array.isArray(R)?R:[R];q._client.postV1projectsTracesByTraceIdTags(q.projectId,D,{tags:P}).catch((U)=>{F.error(`tag failed: ${String(U)}`)})})}static asyncEvaluate(R,f){S("BaseTracer.asyncEvaluate",()=>{let{judge:q,example:$}=R,P=V._getProxyProvider().getActiveTracer();if(!P?.projectId)return;if(!P.supportsLiveInstrumentation)return;let U=V._resolveSpan(f);if(!U?.isRecording())return;let O=P.getSpanProcessor(),E=U.spanContext(),Q=O.stateIncr(E,"pending_evals_count"),Y={project_id:P.projectId,eval_name:`async_evaluate_${q}_${Q}`,judges:[{name:q}],examples:[{...$,example_id:crypto.randomUUID(),created_at:new Date().toISOString(),trace_id:E.traceId,span_id:E.spanId}],is_offline:!1,is_behavior:!1},W=O.stateAppend(E,"pending_evals",Y);U.setAttribute("judgment.pending_trace_eval",JSON.stringify(W))})}}function Bf(R,f){try{let q=Zf(R).map((D)=>D.replace(/^\.\.\./,"").split("=")[0].trim()).filter((D)=>D.length>0),$={};return q.forEach((D,P)=>{if(P<f.length)$[D]=f[P]}),$}catch{return{}}}var d,Af="judgeval";var p=H(()=>{FR();T1();Xf();k();T();I();xR();k1();QR();d=require("@opentelemetry/api")});function N1(){return If.getStore()===!0}function Kf(R){if(L1=R,vf)return;let f=y.context;f.active=()=>{if(!N1())return kq();let q=O1.getStore();if(q)return q;return L1?L1():y.ROOT_CONTEXT},f.with=(q,$,D,...P)=>{if(!N1())return Lq(q,$,D,...P);return O1.run(q,()=>$.apply(D,P))},f.bind=(q,$)=>{if(!N1())return Nq(q,$);if(typeof $!=="function")return $;let D=$;return(...P)=>O1.run(q,()=>D(...P))},vf=!0}function A1(R,f){return If.run(!0,()=>O1.run(R,f))}var y,y1,vf=!1,L1=null,If,O1,kq,Lq,Nq;var uf=H(()=>{y=require("@opentelemetry/api"),y1=require("async_hooks"),If=new y1.AsyncLocalStorage,O1=new y1.AsyncLocalStorage,kq=y.context.active.bind(y.context),Lq=y.context.with.bind(y.context),Nq=y.context.bind.bind(y.context)});class lf{_provider;constructor(R){this._provider=R}startSpan(R,f,q){let $=q??this._provider.getCurrentContext();return this._provider._getDelegateTracer().startSpan(R,f,$)}startActiveSpan(R,...f){let q={},$=this._provider.getCurrentContext(),D;if(f.length===1)D=f[0];else if(f.length===2)q=f[0],D=f[1];else q=f[0],$=f[1],D=f[2];let P=this.startSpan(R,q,$);return this._provider.useSpan(P,!1,!1,!1,()=>D(P))}}class j{static _instance=null;_activeTracer=null;_instrumentations=[];_proxyTracer;_tracers=new Set;constructor(){this._proxyTracer=new lf(this),hf(this),Kf(()=>this.getCurrentContext())}static getInstance(){return j._instance??=new j,j._instance}static installAsGlobalTracerProvider(){let R=j.getInstance();return L.trace.setGlobalTracerProvider(R)}register(R){this._tracers.add(R)}deregister(R){this._tracers.delete(R)}setActive(R){let f=this.getCurrentSpan();if(f?.isRecording()){if(L.trace.getSpan(this.getCurrentContext())===f)return F.error("Cannot set_active() while a root span is active. Keeping existing tracer provider."),!1}return this.register(R),this._activeTracer=R,!0}getActiveTracer(){return this._activeTracer}getCurrentContext(){return V1.getStore()??L.ROOT_CONTEXT}setSpan(R,f){return L.trace.setSpan(R,f)}wrapSpanContext(R){return L.trace.wrapSpanContext(R)}getCurrentSpan(){let R=this.getCurrentContext();return L.trace.getSpan(R)}hasActiveRootSpan(){if(!this.getCurrentSpan()?.isRecording())return!1;return!0}_getDelegateTracer(){let R=this._activeTracer;if(!R)return F.debug("No active tracer, returning NoOpTracer"),R1;return R._tracerProvider.getTracer(yq)}getTracer(R,f,q){return this._proxyTracer}addInstrumentation(R){try{df.registerInstrumentations({tracerProvider:this,instrumentations:[R]}),this._instrumentations.push(R)}catch(f){F.error(`Failed to add instrumentation: ${String(f)}`)}}useSpan(R,f,q,$,D){let P=this.getCurrentContext(),U=L.trace.setSpan(P,R);return V1.run(U,()=>A1(U,()=>{try{let O=D();if(O instanceof Promise)return O.catch((E)=>{if(R.isRecording()){if(q)R.recordException(E);if($){let Q=E;R.setStatus({code:L.SpanStatusCode.ERROR,message:`${Q.name}: ${Q.message}`})}}throw E}).finally(()=>{if(f)R.end()});if(f)R.end();return O}catch(O){if(R.isRecording()){if(q)R.recordException(O);if($){let E=O;R.setStatus({code:L.SpanStatusCode.ERROR,message:`${E.name}: ${E.message}`})}}if(f)R.end();throw O}}))}attachContext(R){V1.enterWith(R)}withContext(R,f){return V1.run(R,()=>A1(R,f))}async forceFlush(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.forceFlush()));for(let f of R)if(f.status==="rejected")F.error(`forceFlush failed: ${String(f.reason)}`)}async shutdown(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.shutdown()));for(let f of R)if(f.status==="rejected")F.error(`shutdown failed: ${String(f.reason)}`);this._activeTracer=null,this._tracers.clear()}}var L,df,pf,yq="judgeval",V1;var kR=H(()=>{T();uf();z1();QR();L=require("@opentelemetry/api"),df=require("@opentelemetry/instrumentation"),pf=require("async_hooks"),V1=new pf.AsyncLocalStorage});class A{_delegate;constructor(R,f,q,$){if(!R){this._delegate=null;return}this._delegate=new sf.OTLPTraceExporter({url:R,headers:{Authorization:`Bearer ${f}`,"X-Organization-Id":q,"X-Project-Id":$}})}export(R,f){F.info(`Exported ${R.length} spans`),this._delegate?.export(R,f)}shutdown(){return this._delegate?.shutdown()??Promise.resolve()}forceFlush(){return this._delegate?.forceFlush()??Promise.resolve()}}var sf;var LR=H(()=>{T();sf=require("@opentelemetry/exporter-trace-otlp-http")});var nf,n;var F1=H(()=>{LR();nf=require("@opentelemetry/core");n=class n extends A{constructor(){super("","","","")}export(R,f){f({code:nf.ExportResultCode.SUCCESS})}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}}});class YR{_keyPredicate;constructor(R=Q1){this._keyPredicate=R}onStart(R,f){let q=s(f)?.getAllEntries()??[];for(let[$,D]of q)if(this._keyPredicate($))R.setAttribute($,D.value)}onEnd(R){}forceFlush(){return Promise.resolve()}shutdown(){return Promise.resolve()}}var Q1=()=>!0;var B1=H(()=>{xR()});function WR(R){return`${R.traceId}:${R.spanId}`}function Aq(R){return R[0]===0&&R[1]===0}var gf,g;var NR=H(()=>{FR();k();QR();B1();gf=require("@opentelemetry/sdk-trace-base");g=class g extends gf.BatchSpanProcessor{tracer;_state=new Map;_spanFinalizers;_baggageProcessor;constructor(R,f,q){super(f,q);this.tracer=R,this._spanFinalizers=new FinalizationRegistry(($)=>{this._cleanupSpanState($)}),this._baggageProcessor=new YR}_cleanupSpanState(R){this._state.delete(R)}_registerSpan(R){let f=R.spanContext();if(!f.traceId||!f.spanId)return;let q=WR(f);this._spanFinalizers.register(R,q)}stateSet(R,f,q){let $=WR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);D.set(f,q)}stateGet(R,f,q){let $=WR(R),D=this._state.get($);if(!D?.has(f))return q;return D.get(f)}stateIncr(R,f){let q=WR(R),$=this._state.get(q);if(!$)$=new Map,this._state.set(q,$);let D=$.get(f),P=typeof D==="number"?D:0;return $.set(f,P+1),P}stateAppend(R,f,q){let $=WR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);let P=D.get(f),U=Array.isArray(P)?[...P,q]:[q];return D.set(f,U),U}_emitSpan(R,f=!1){let q=R.spanContext();if(!q.traceId)return;let $=this.stateIncr(q,"judgment.update_id"),D={...R.attributes,"judgment.update_id":$};if(f)delete D["judgment.pending_trace_eval"];let P=Object.create(R);Object.defineProperty(P,"attributes",{value:D,writable:!1});let U=Aq(R.endTime)?R.startTime:R.endTime;Object.defineProperty(P,"endTime",{value:U,writable:!1}),super.onEnd(P)}emitPartial(){S("JudgmentSpanProcessor.emitPartial",()=>{let R=u().getCurrentSpan();if(!R?.isRecording())return;let f=R.spanContext();if(!f.traceId)return;if(this.stateGet(f,"disable_partial_emit",!1))return;this._emitSpan(R,!0)})}onStart(R,f){S("JudgmentSpanProcessor.onStart",()=>{this._baggageProcessor.onStart(R,f),this._registerSpan(R)})}onEnd(R){S("JudgmentSpanProcessor.onEnd",()=>{let f=R.spanContext();if(!f.traceId){super.onEnd(R);return}let q=WR(f);try{if(!this.stateGet(f,"cancelled",!1))this._emitSpan(R)}finally{this._cleanupSpanState(q)}})}}});var HR;var v1=H(()=>{F1();NR();HR=class HR extends g{constructor(){super(null,new n)}onStart(R,f){}onEnd(R){}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}emitPartial(){}stateSet(R,f,q){}stateGet(R,f,q){return q}stateIncr(R,f){return 0}stateAppend(R,f,q){return[q]}}});var Y1,I1,B;var W1=H(()=>{GR();IR();T();uR();I();w1();p();kR();LR();F1();NR();v1();Y1=require("@opentelemetry/resources"),I1=require("@opentelemetry/sdk-trace-node");B=class B extends V{_spanExporter=null;_spanProcessor=null;constructor(R,f,q,$,D,P,U,O,E,Q){super(R,f,q,$,D,P,U,O,E,Q)}static async init(R={}){let f=R.apiKey??qR,q=R.organizationId??$R,$=R.apiUrl??DR,D=R.projectName??null,P=R.serializer??w,U=!0;if(!D)F.warning("project_name not provided. Tracer will not export spans."),U=!1;if(!f)F.warning("api_key not provided. Tracer will not export spans."),U=!1;if(!q)F.warning("organization_id not provided. Tracer will not export spans."),U=!1;if(!$)F.warning("api_url not provided. Tracer will not export spans."),U=!1;let O=null,E=null;if(U&&D&&f&&q&&$){if(O=new c($,f,q),E=await PR(O,D).catch(()=>null),!E)F.warning(`Project '${D}' not found. Tracer will not export spans.`),U=!1}let Q={"service.name":D??"unknown","telemetry.sdk.name":"judgeval","telemetry.sdk.version":tR};if(R.environment)Q["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(Q,R.resourceAttributes);let Y=Y1.defaultResource().merge(Y1.resourceFromAttributes(Q)),W=new I1.NodeTracerProvider({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits}),C=new B(D,E,f,q,$,R.environment??null,P,W,O,U);if(U){let h=new I1.NodeTracerProvider({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[C.getSpanProcessor(),...R.spanProcessors??[]]});C._tracerProvider=h}if(j.getInstance().register(C),R.setActive??!0)C.setActive();return C}getSpanExporter(){if(this._spanExporter)return this._spanExporter;if(!this._enableMonitoring||!this.projectId||!this.apiKey||!this.organizationId||!this.apiUrl)this._spanExporter=new n;else{let R=this.apiUrl.endsWith("/")?this.apiUrl+"otel/v1/traces":this.apiUrl+"/otel/v1/traces";this._spanExporter=new A(R,this.apiKey,this.organizationId,this.projectId)}return this._spanExporter}getSpanProcessor(){if(this._spanProcessor)return this._spanProcessor;if(!this._enableMonitoring)this._spanProcessor=new HR;else this._spanProcessor=new g(this,this.getSpanExporter());return this._spanProcessor}}});var SR;var K1=H(()=>{wR();NR();SR=class SR extends g{_dataset;_exampleFields;_seenTraceIds=new Set;constructor(R,f,q){super(R,f);this._dataset=q.dataset,this._exampleFields={...q.exampleFields??{}}}_maybeCreateExample(R){if(R.parentSpanContext)return;let f=R.spanContext();if(!f?.traceId)return;if(this._seenTraceIds.has(f.traceId))return;this._seenTraceIds.add(f.traceId);let q=x.create({...this._exampleFields,offline_trace_id:f.traceId});this._dataset.push(q)}onEnd(R){try{this._maybeCreateExample(R)}finally{super.onEnd(R)}}}});var tf={};BR(tf,{OfflineTracer:()=>i});var H1,u1,of="otel/v1/offline-traces",i;var S1=H(()=>{GR();IR();uR();I();w1();kR();W1();LR();K1();H1=require("@opentelemetry/resources"),u1=require("@opentelemetry/sdk-trace-node");i=class i extends B{supportsLiveInstrumentation=!1;_offlineApiUrl;_offlineApiKey;_offlineOrganizationId;_offlineProjectId;_dataset;_exampleFields;_offlineSpanExporter=null;_offlineSpanProcessor=null;constructor(R){super(R.projectName,R.projectId,R.apiKey,R.organizationId,R.apiUrl,R.environment,R.serializer,R.tracerProvider,R.client,!0);this._offlineApiUrl=R.apiUrl,this._offlineApiKey=R.apiKey,this._offlineOrganizationId=R.organizationId,this._offlineProjectId=R.projectId,this._dataset=R.dataset,this._exampleFields=R.exampleFields}static async create(R){let f=R.apiKey??qR,q=R.organizationId??$R,$=R.apiUrl??DR,D=R.projectName,P=R.serializer??w;if(!D)throw Error("projectName is required for OfflineTracer");if(!f)throw Error("apiKey is required for OfflineTracer");if(!q)throw Error("organizationId is required for OfflineTracer");if(!$)throw Error("apiUrl is required for OfflineTracer");let U=new c($,f,q),O;try{O=await PR(U,D)}catch(G){throw Error(`Project '${D}' not found; cannot start OfflineTracer: ${String(G)}`)}let E={"service.name":D,"telemetry.sdk.name":"judgeval","telemetry.sdk.version":tR,"judgment.offline":"true"};if(R.environment)E["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(E,R.resourceAttributes);let Q=H1.defaultResource().merge(H1.resourceFromAttributes(E)),Y=new i({projectName:D,projectId:O,apiKey:f,organizationId:q,apiUrl:$,environment:R.environment??null,serializer:P,tracerProvider:new u1.NodeTracerProvider({resource:Q}),client:U,dataset:R.dataset,exampleFields:{...R.exampleFields??{}}}),W=new u1.NodeTracerProvider({resource:Q,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[Y.getSpanProcessor(),...R.spanProcessors??[]]});if(Y._tracerProvider=W,j.getInstance().register(Y),R.setActive??!0)Y.setActive();return Y}getSpanExporter(){if(this._offlineSpanExporter)return this._offlineSpanExporter;let R=this._offlineApiUrl.endsWith("/")?this._offlineApiUrl+of:this._offlineApiUrl+"/"+of;return this._offlineSpanExporter=new A(R,this._offlineApiKey,this._offlineOrganizationId,this._offlineProjectId),this._offlineSpanExporter}getSpanProcessor(){if(this._offlineSpanProcessor)return this._offlineSpanProcessor;return this._offlineSpanProcessor=new SR(this,this.getSpanExporter(),{dataset:this._dataset,exampleFields:this._exampleFields}),this._offlineSpanProcessor}}});var lq={};BR(lq,{wrapOpenAI:()=>rR,wrap:()=>eR,validateDatasetSchema:()=>gR,propagation:()=>E1,inferSchemaFromExamples:()=>iR,baggage:()=>D1,Tracer:()=>B,OfflineTracer:()=>i,OfflineTestsFactory:()=>ZR,OfflineTestRunner:()=>CR,OfflineJudgmentSpanProcessor:()=>SR,NoOpSpanProcessor:()=>HR,NoOpSpanExporter:()=>n,JudgmentTracerProvider:()=>j,JudgmentSpanExporter:()=>A,JudgmentBaggageSpanProcessor:()=>YR,JudgmentBaggagePropagator:()=>l,Judgeval:()=>X1,Judge:()=>UR,Example:()=>x,Evaluation:()=>OR,Dataset:()=>r,BaseTracer:()=>V,AgentJudgeFactory:()=>VR,ALLOW_ALL_BAGGAGE_KEYS:()=>Q1});module.exports=Uq(lq);GR();IR();uR();T();class UR{}T();var h1=JR(hR(),1);T();var J=JR(hR(),1),Qq=2000;class ER{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async _poll(R,f,q,$){let D=Date.now();while(!0){let P=(Date.now()-D)/1000;if(P>$)throw Error(`Evaluation timed out after ${$}s`);let U=await this._client.getV1projectsExperimentsByRunId(R,f),O=U.results??[];if(O.length>=q){let Q=U.ui_results_url??"Failed to get UI results URL";return console.log(`${J.default.green("✓")} Evals completed and saved in ${J.default.bold(`${P.toFixed(1)}s`)}`),{results:O,url:Q}}await new Promise((Q)=>setTimeout(Q,Qq))}}_displayResults(R,f,q,$){let D=[],P=0,U=0;console.log();for(let O=0;O<f.length;O++){let E=f[O],Q=E.scorers.every((Y)=>Boolean(Y.success));if(Q)P++,console.log(`${J.default.green("✓")} Example ${O+1}: ${J.default.green("PASSED")}`);else U++,console.log(`${J.default.red("✗")} Example ${O+1}: ${J.default.red("FAILED")}`);for(let Y of E.scorers){let W=Y.score!==null?Y.score.toFixed(3):"N/A",C=Y.success?J.default.green(W):J.default.red(W);console.log(`  ${J.default.dim(`${Y.name}:`)} ${C} ${J.default.dim(`(threshold: ${Y.threshold})`)}`)}D.push({success:Q,scorers:E.scorers,example:R[O]})}if(console.log(),P===D.length)console.log(`${J.default.bold(J.default.green("✓ All tests passed!"))} (${P}/${D.length})`);else console.log(`${J.default.bold(J.default.yellow("⚠ Results:"))} ${J.default.green(`${P} passed`)} | ${J.default.red(`${U} failed`)}`);if(console.log(`${J.default.dim("View full details:")} ${J.default.underline(q)}`),console.log(),$&&D.some((O)=>!O.success)){let O=[`Evaluation failed: ${U}/${D.length} examples failed`];for(let E=0;E<D.length;E++)if(!D[E].success){O.push(`  Example ${E+1}:`);for(let Q of D[E].scorers)if(!Q.success){if(O.push(`    ${Q.name}: ${Q.score!==null?Q.score.toFixed(3):"N/A"} (threshold: ${Q.threshold})`),Q.reason)O.push(`      ${Q.reason}`)}}throw Error(O.join(`
+`))}return D}async run(R,f,q,$=!1,D=300){if(!this._projectId)return F.error("Project ID is not resolved. Evaluation requires a valid project."),[];let P=this._projectId,U=crypto.randomUUID(),O=new Date().toISOString();console.log(),console.log(J.default.bold(J.default.cyan("Starting Evaluation"))),console.log(`${J.default.dim("Run:")} ${q}`),console.log(`${J.default.dim("Project:")} ${this._projectName}`),console.log(`${J.default.dim("Examples:")} ${R.length} | ${J.default.dim("Scorers:")} ${f.length}`),console.log();let E=this._buildPayload(U,P,q,O,R,f),Q=await this._submit(P,U,R,f,E),{results:Y,url:W}=await this._poll(P,U,Q,D);return this._displayResults(R,Y,W,$)}}class lR extends ER{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:[],custom_scorers:[]}}async _submit(R,f,q,$,D){let P=Date.now(),U=q.flatMap((W,C)=>$.map((G)=>G.score(W).then((h)=>({exampleIdx:C,scorer:G,result:h,error:null})).catch((h)=>({exampleIdx:C,scorer:G,result:null,error:String(h)})))),O=await Promise.all(U),E=((Date.now()-P)/1000).toFixed(1);console.log(`${h1.default.green("✓")} Scoring completed in ${h1.default.bold(`${E}s`)}`);let Q=new Map;for(let W of O){let C=Q.get(W.exampleIdx);if(!C)C=[],Q.set(W.exampleIdx,C);C.push(W)}let Y=q.map((W,C)=>{return{scorers_data:(Q.get(C)??[]).map((h)=>{if(h.error!==null)return{scorer_name:h.scorer.constructor.name,value:0,reason:"",error:h.error};let X=h.result;return{scorer_name:h.scorer.constructor.name,value:X.value,reason:X.reason,...X.citations&&{citations:X.citations.map((Z)=>({span_id:Z.spanId,span_attribute:Z.spanAttribute}))}}}),data_object:W.toJSON()}});return await this._client.postV1projectsEvalResultsExamples(R,{results:Y,run:D}),q.length}}var a1=JR(hR(),1);class sR extends ER{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:P.map((U)=>({name:U})),custom_scorers:[]}}async _submit(R,f,q,$,D){return await this._client.postV1projectsEvalQueueExamples(R,D),console.log(`${a1.default.green("✓")} Evaluation submitted`),q.length}}class OR{_local;_hosted;constructor(R,f,q){this._local=new lR(R,f,q),this._hosted=new sR(R,f,q)}run(R){let{examples:f,scorers:q,evalRunName:$,assertTest:D=!1,timeoutSeconds:P=300}=R,U=q.filter((E)=>typeof E==="string"),O=q.filter((E)=>E instanceof UR);if(O.length>0&&U.length>0)return F.error("Running both local and hosted scorers is not supported. Please run your evaluation with either local or hosted scorers, but not both."),Promise.resolve([]);if(O.length===0&&U.length===0)return F.error("No valid local or hosted scorers provided. Please provide at least one local or hosted scorer."),Promise.resolve([]);if(O.length>0)return this._local.run(f,O,$,D,P);return this._hosted.run(f,U,$,D,P)}}class nR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}create(){return new OR(this._client,this._projectId,this._projectName)}}wR();T();wR();class r{name;projectId;projectName;datasetKind;examples;_client;constructor(R){this.name=R.name,this.projectId=R.projectId,this.projectName=R.projectName,this.datasetKind=R.datasetKind??"example",this.examples=R.examples??[],this._client=R.client??null}async addExamples(R,f=100){if(!this._client)return;for(let q=0;q<R.length;q+=f){let $=R.slice(q,q+f);await this._client.postV1projectsDatasetsByDatasetIdentifierExamples(this.projectId,this.name,{examples:$.map((D)=>D.toJSON())})}}async addFromJson(R,f=100){let{readFile:q}=await import("fs/promises"),$=await q(R,"utf-8"),P=JSON.parse($).map((U)=>{if(typeof U!=="object"||U===null)throw Error("Each item in the JSON array must be an object");return x.create(U)});await this.addExamples(P,f)}get length(){return this.examples.length}[Symbol.iterator](){return this.examples[Symbol.iterator]()}toString(){return`Dataset(name=${this.name}, examples=${this.examples.length})`}}var Yq="trace";function gR(R){if(typeof R!=="object"||R===null||Array.isArray(R))throw Error("Dataset schema must be a JSON object.");let f=R;if(f.type!=="object")throw Error('Dataset schema must declare top-level type "object".');let q=f.properties;if(typeof q!=="object"||q===null||Array.isArray(q))throw Error("Dataset schema must declare a 'properties' object.");let $=Object.entries(q).filter(([,D])=>typeof D==="object"&&D!==null&&D.type===Yq).map(([D])=>D);if($.length>1)throw Error(`A dataset may declare at most one trace column; found ${$.length}: ${$.join(", ")}.`)}function Wq(R){if(typeof R==="boolean")return"boolean";if(typeof R==="number")return Number.isInteger(R)?"integer":"number";if(typeof R==="string")return"string";if(Array.isArray(R))return"array";if(typeof R==="object"&&R!==null)return"object";return"string"}function iR(R){if(R.length===0)throw Error("Cannot infer a dataset schema without examples. Pass an explicit `schema` to client.datasets.create().");let f={},q=null;for(let $ of R){let D=new Set;for(let[P,U]of Object.entries($.properties)){if(U===null||U===void 0)continue;if(D.add(P),!(P in f))f[P]={type:Wq(U)}}if(q===null)q=D;else if(q.size!==D.size||![...q].every((P)=>D.has(P)))throw Error("All examples must share the same set of fields (dataset schemas require all declared properties). Pass an explicit `schema` or make all examples homogeneous.")}return{type:"object",properties:f}}class oR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async get(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsDatasetsByDatasetIdentifier(f,R),$=q.dataset_kind??"example",D=(q.examples??[]).map((P)=>x.from({...P.data??{},example_id:P.example_id,created_at:P.created_at??""}));return new r({name:R,projectId:f,projectName:this._projectName,datasetKind:$,examples:D,client:this._client})}async create(R,f={}){let q=this._expectProjectId();if(!q)return null;let{examples:$=[],overwrite:D=!1,batchSize:P=100,schema:U}=f,O;if(U!==void 0)gR(U),O=U;else{if($.length===0)throw Error("Datasets require a JSON Schema. Pass `schema` to create(), or provide `examples` to infer one.");O=iR($)}await this._client.postV1projectsDatasets(q,{name:R,examples:[],dataset_kind:"example",overwrite:D,schema:O});let E=new r({name:R,projectId:q,projectName:this._projectName,examples:$,client:this._client});if($.length>0)await E.addExamples($,P);return E}list(){let R=this._expectProjectId();if(!R)return Promise.resolve(null);return this._client.getV1projectsDatasets(R)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Dataset operations require a valid project."),null;return this._projectId}}T();class VR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async create(R){let f=this._expectProjectId();if(!f)return null;let q={name:R.name,prompt:R.prompt,model:R.model,score_type:R.scoreType};if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;return{judgeId:(await this._client.postV1projectsJudges(f,q)).judge_id,name:R.name,prompt:R.prompt,model:R.model,scoreType:R.scoreType,description:R.description??null,judgeDescription:R.judgeDescription??null,categories:R.categories??null,minScore:R.minScore??null,maxScore:R.maxScore??null,majorVersion:0,minorVersion:0}}async update(R){let f=this._expectProjectId();if(!f)return null;let q={};if(R.prompt!==void 0)q.prompt=R.prompt;if(R.model!==void 0)q.model=R.model;if(R.scoreType!==void 0)q.score_type=R.scoreType;if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;if(R.sourceMajorVersion!==void 0)q.source_major_version=R.sourceMajorVersion;if(R.sourceMinorVersion!==void 0)q.source_minor_version=R.sourceMinorVersion;if(R.targetMajorVersion!==void 0)q.target_major_version=R.targetMajorVersion;if(R.targetMinorVersion!==void 0)q.target_minor_version=R.targetMinorVersion;let $=await this._client.patchV1projectsJudgesByJudgeId(f,R.judgeId,q);return Hq($)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Agent judge operations require a valid project."),null;return this._projectId}}function Hq(R){let f=R.judge;return{judgeId:f.id,name:f.name,prompt:f.prompt??"",model:f.model??"",scoreType:f.score_type,description:f.description??null,judgeDescription:f.judge_description??null,categories:f.categories??null,minScore:f.min_score??null,maxScore:f.max_score??null,majorVersion:f.major_version??null,minorVersion:f.minor_version??null}}T();T();W1();S1();kR();function C1(R){let f=Array.isArray(R.judges)?R.judges.filter((q)=>typeof q==="object"&&q!==null):[];return{id:typeof R.id==="string"?R.id:"",name:typeof R.name==="string"?R.name:"",datasetId:typeof R.dataset_id==="string"?R.dataset_id:"",description:typeof R.description==="string"?R.description:null,createdAt:typeof R.created_at==="string"?R.created_at:null,judges:f}}function cf(R){let f=[];for(let q of R)for(let $ of q.scorers)if($.success!==null&&$.success!==void 0)f.push($.success);if(f.length===0)return null;return f.every((q)=>q)}var _=JR(hR(),1);function m(R){return typeof R==="object"&&R!==null&&!Array.isArray(R)?R:{}}function RR(R){return Array.isArray(R)?R:[]}function b(R){return typeof R==="string"?R:""}function rf(R){return R?"Yes":"No"}function Bq(R){let{score_type:f,bool_value:q,str_value:$,num_value:D}=R;if(f==="binary")return typeof q==="boolean"?rf(q):null;if(f==="categorical")return typeof $==="string"?$:null;if(f==="numeric")return typeof D==="number"?D:null;if(typeof q==="boolean")return rf(q);if(typeof $==="string")return $;if(typeof D==="number")return D;return null}function vq(R){if(typeof R==="object"&&R!==null){let f=R.text;return typeof f==="string"&&f?f:null}if(typeof R==="string"){try{let f=JSON.parse(R);if(typeof f==="object"&&f!==null){let q=f.text;if(typeof q==="string")return q||null}}catch{}return R||null}return null}function ef(R){if(!R||R.length===0)return;let f=[];for(let q of R){if(!q.name&&!q.judgeId)throw Error("judgeVersions entries require a 'name' (or 'judgeId') key");let $={};if(q.judgeId!==void 0)$.judge_id=q.judgeId;if(q.name!==void 0)$.name=q.name;if(q.tag!==void 0)$.tag=q.tag;if(q.version!==void 0)$.version=q.version;if(q.majorVersion!==void 0)$.major_version=q.majorVersion;if(q.minorVersion!==void 0)$.minor_version=q.minorVersion;f.push($)}return f}function af(R,f,q){let $=[];for(let D of R){let P=b(D.example_id),U=m(D.data),O=[];for(let E of RR(D.scorers)){let Q=m(E),Y={judge_id:Q.judge_id,judge_major_version:Q.judge_major_version,judge_minor_version:Q.judge_minor_version},W=vq(Q.reason);if(W)Y.reason=W;O.push({name:b(Q.judge_name),value:Bq(Q),scoreType:typeof Q.score_type==="string"?Q.score_type:null,error:typeof Q.error==="string"?Q.error:null,success:typeof Q.success==="boolean"?Q.success:null,metadata:Y})}if(q){let E=Boolean(q({...U},O));for(let Q of O)Q.success=E}$.push({exampleId:P,data:U,scorers:O,agentOfflineTraceId:f[P]})}return $}function Rq(R){if(R.status!=="completed")throw Error(`Test run ${R.testRunId} finished with status '${R.status}'`);let f=R.results.filter((q)=>q.scorers.some(($)=>$.success===!1)).map((q)=>q.exampleId);if(f.length>0||R.passed!==!0)throw Error(`Test run ${R.testRunId} failed its pass condition for ${f.length} example(s): ${JSON.stringify(f)}`)}function fq(R,f,q){console.log(),console.log(_.default.bold(_.default.cyan("Starting Offline Test"))),console.log(`${_.default.dim("Config:")} ${R}`),console.log(`${_.default.dim("Project:")} ${f}`),console.log(`${_.default.dim("Examples:")} ${q}`),console.log()}function qq(R,f){if(console.log(),R.forEach((q,$)=>{if(q.scorers.some((P)=>P.success!==null&&P.success!==void 0)){let P=q.scorers.every((U)=>U.success===!0);console.log(P?`${_.default.green("✓")} Example ${$+1}: ${_.default.green("PASSED")}`:`${_.default.red("✗")} Example ${$+1}: ${_.default.red("FAILED")}`)}else console.log(`${_.default.cyan("•")} Example ${$+1}:`);for(let P of q.scorers){if(P.error){console.log(`  ${_.default.dim(`${P.name}:`)} ${_.default.red(`error: ${P.error}`)}`);continue}let U=P.value===null?"N/A":String(P.value),O=P.success===!0?_.default.green(U):P.success===!1?_.default.red(U):U;console.log(`  ${_.default.dim(`${P.name}:`)} ${O}`)}}),console.log(),f)console.log(`${_.default.dim("View full details:")} ${_.default.underline(f)}`),console.log()}var Iq=new Set(["completed","error","cancelled"]),Kq=100,uq=200,dq=2000,Z1=1e4;class CR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async resolveDatasetVersion(R,f){let q=await this._client.getV1projectsDatasetsByDatasetIdentifierVersions(this._projectId,R.datasetId),$=RR(q.versions).filter((D)=>typeof D==="object"&&D!==null).map((D)=>D);if(f===void 0){if($.length===0)throw Error(`Dataset of test config '${R.name}' has no versions`);return $.reduce((D,P)=>Number(P.version_number??0)>Number(D.version_number??0)?P:D)}if(typeof f==="number"){let D=$.find((P)=>Number(P.version_number??0)===f);if(D)return D}else{let D=$.find((P)=>P.version_id===f);if(D)return D}throw Error(`Dataset version ${JSON.stringify(f)} does not exist for the dataset of test config '${R.name}'`)}async fetchExamples(R,f){let q=[],$,D,P=0;for(;;){if(P>=Z1)throw Error(`fetchExamples exceeded ${Z1} pages for dataset of test config '${R.name}'`);let U=await this._client.getV1projectsDatasetsByDatasetIdentifierPage(this._projectId,R.datasetId,String(f),String(Kq),$,D);P+=1;for(let Q of RR(U.entries)){let Y=m(m(Q).example),W=Y.data;if(typeof W==="string")try{W=JSON.parse(W)}catch{W={}}q.push({exampleId:b(Y.example_id),data:m(W),offlineTraceId:typeof Y.offline_trace_id==="string"?Y.offline_trace_id:null,createdAt:typeof Y.created_at==="string"?Y.created_at:null})}let O=m(U.metadata),E=m(O.nextCursor);if(!O.hasMore||Object.keys(E).length===0)break;$=b(E.created_at),D=b(E.example_id)}return q}createTestRun(R,f={}){let q={test_config_id:R.id,source:f.source??"sdk"};if(f.name)q.name=f.name;if(typeof f.datasetVersion==="number")q.dataset_version_number=f.datasetVersion;else if(typeof f.datasetVersion==="string")q.dataset_version_id=f.datasetVersion;let $=ef(f.judgeVersions);if($)q.judge_versions=$;let D=f.agentTraces??{};if(Object.keys(D).length>0)q.agent_traces=Object.entries(D).map(([P,U])=>({example_id:P,agent_offline_trace_id:U}));return this._client.postV1projectsTestRuns(this._projectId,q)}async runAgent(R,f){let q=[],$=j.getInstance().getActiveTracer();await i.create({projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl(),dataset:q,setActive:!0});let D={};try{let P=B.observe(R,{spanType:"agent"});for(let U of f){let O=q.length;try{await P(U.data)}catch(E){F.error(`Agent entrypoint raised for example ${U.exampleId}: ${String(E)}`)}for(let E of q.slice(O)){let Q=E.get("offline_trace_id");if(U.exampleId&&typeof Q==="string"){D[U.exampleId]=Q;break}}}}finally{if(await B.forceFlush(),$)$.setActive()}return D}async waitForCompletion(R,f){let q=Date.now();for(;;){if((Date.now()-q)/1000>f)throw Error(`Test run ${R} did not complete within ${f}s`);let $=await this._client.getV1projectsTestRunsByTestRunId(this._projectId,R),D=b(m($.test_run).status);if(Iq.has(D))return D;await KR(dq)}}async fetchItems(R){let f=[],q="",$,D=0;for(;;){if(D>=Z1)throw Error(`fetchItems exceeded ${Z1} pages for test run ${R}`);let P=await this._client.getV1projectsTestRunsByTestRunIdItems(this._projectId,R,String(uq),$);D+=1;for(let O of RR(P.results))f.push(m(O));if(!q)q=b(P.ui_results_url);let U=P.next_cursor;if(!P.has_more||typeof U!=="string"||!U)break;$=U}return{items:f,uiResultsUrl:q}}async reportSuccess(R,f,q,$){let D=new Map,P=new Map,U=(Y,W,C,G)=>`${Y}\x00${W}\x00${C}\x00${G}`,O=(Y,W)=>`${Y}\x00${W}`;for(let Y of RR(f.evaluation_runs)){let W=m(Y),C=W.run_id;if(typeof C!=="string"||!C)continue;D.set(U(b(W.example_id),b(W.judge_id),Number(W.judge_major_version??0),Number(W.judge_minor_version??0)),C),P.set(O(b(W.example_id),b(W.judge_name)),C)}let E=new Map;for(let Y of $)E.set(Y.exampleId,Y);let Q=[];for(let Y of q){let W=b(Y.example_id),C=E.get(W),G=C?C.scorers.map((X)=>X.success??null):[],h=RR(Y.scorers);for(let X=0;X<h.length;X+=1){let Z=m(h[X]),M=D.get(U(W,b(Z.judge_id),Number(Z.judge_major_version??0),Number(Z.judge_minor_version??0)))??P.get(O(W,b(Z.judge_name)));if(!M){F.warning(`No evaluation run ref for scorer ${b(Z.judge_name)} of example ${W}; skipping its success update`);continue}Q.push({evaluation_run_id:M,success:X<G.length?G[X]:null})}}if(Q.length===0)return;await this._client.patchV1projectsTestRunsByTestRunIdSuccess(this._projectId,R,{successes:Q})}async run(R,f={}){let{agentFunction:q,judgeVersions:$,datasetVersion:D,passConditionFn:P,assertTest:U=!1,timeoutSeconds:O=600,runName:E}=f;if(U&&!P)throw Error("assertTest=true requires a passConditionFn to decide whether each row passes.");let Q=await this.resolveDatasetVersion(R,D),Y=Number(Q.version_number??0),W=await this.fetchExamples(R,Y);fq(R.name,this._projectName,W.length),F.info(`Dataset version ${Y}: ${W.length} example(s)`);let C=typeof D==="string"?D:Y,G={};if(q&&W.length>0)G=await this.runAgent(q,W);let h=await this.createTestRun(R,{datasetVersion:C,judgeVersions:$,agentTraces:G,name:E}),X=m(h.test_run),Z=b(X.id),M=b(h.ui_results_url);F.info(`Created test run ${Z} over ${W.length} examples`);let yR=await this.waitForCompletion(Z,O),{items:AR,uiResultsUrl:v}=await this.fetchItems(Z);if(v)M=v;let o=af(AR,G,P);if(P)await this.reportSuccess(Z,h,AR,o);qq(o,M||void 0);let fR={testRunId:Z,status:yR,uiResultsUrl:M||void 0,results:o,agentOfflineTraceIds:G,passed:cf(o)};if(U)Rq(fR);return fR}}var pq=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function d1(R){return pq.test(R)}class ZR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async createConfig(R,f,q,$){let D=this._expectProjectId();if(!D)return null;let P=[];for(let E of q)if(typeof E==="string")P.push(d1(E)?{judge_id:E}:{name:E});else if(E.judgeId)P.push({judge_id:E.judgeId});else if(E.name)P.push({name:E.name});else throw Error("judges entries must be judge names or objects with a 'judgeId' or 'name'");let U={name:R,judges:P};if($!==void 0)U.description=$;if(d1(f))U.dataset_id=f;else U.dataset_name=f;let O=await this._client.postV1projectsTestConfigs(D,U);return F.info(`Created test config ${R}`),C1(O.test_config??{})}async getConfig(R){let f=this._expectProjectId();if(!f)return null;if(d1(R)){let D=await this._client.getV1projectsTestConfigsByTestConfigId(f,R);return C1(D.test_config??{})}let $=(await this.listConfigs()??[]).find((D)=>D.name===R);if(!$)return F.error(`Test config '${R}' not found`),null;return $}async listConfigs(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsTestConfigs(f,R);return(Array.isArray(q.test_configs)?q.test_configs:[]).map((D)=>C1(D))}async deleteConfig(R){let f=this._expectProjectId();if(!f)return!1;return await this._client.deleteV1projectsTestConfigsByTestConfigId(f,R),F.info(`Deleted test config ${R}`),!0}async run(R,f={}){let q=this._expectProjectId();if(!q)return null;let $=typeof R==="string"?await this.getConfig(R):R;if(!$||!$.id)return null;return new CR(this._client,q,this._projectName).run($,f)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Offline-test operations require a valid project."),null;return this._projectId}}class X1{_client;_projectName;_projectId;constructor(R,f,q){this._client=R,this._projectName=f,this._projectId=q}static async create(R){let f=R.apiKey??qR,q=R.organizationId??$R,$=R.apiUrl??DR;if(!f)throw Error("API key is required");if(!q)throw Error("Organization ID is required");if(!$)throw Error("API URL is required");if(!R.projectName)throw Error("Project name is required");let D=new c($,f,q),P=null;try{P=await PR(D,R.projectName)}catch{F.warning(`Project '${R.projectName}' not found. Some operations requiring project_id will be skipped.`)}return new X1(D,R.projectName,P)}async offlineTracer(R){let{OfflineTracer:f}=await Promise.resolve().then(() => (S1(),tf));return f.create({...R,projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl()})}get datasets(){return new oR(this._client,this._projectId,this._projectName)}get evaluation(){return new nR(this._client,this._projectId,this._projectName)}get agentJudges(){return new VR(this._client,this._projectId,this._projectName)}get offlineTests(){return new ZR(this._client,this._projectId,this._projectName)}}p();LR();F1();kR();NR();v1();K1();B1();$1();xR();k1();W1();S1();T1();wR();
+
+//# debugId=58E1B30396F5763264756E2164756E21
+//# sourceMappingURL=index.cjs.map