judgeval 1.2.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/dist/node/index.cjs +4 -4
  2. package/dist/node/index.cjs.map +3 -3
  3. package/dist/node/index.mjs +5 -5
  4. package/dist/node/index.mjs.map +3 -3
  5. package/dist/trace/BaseTracer.d.ts +7 -0
  6. package/dist/trace/BaseTracer.d.ts.map +1 -1
  7. package/dist/workers/index.mjs +4 -4
  8. package/dist/workers/index.mjs.map +3 -3
  9. package/package.json +1 -1
package/dist/node/index.mjs CHANGED
@@ -1,7 +1,7 @@
1
- import{createRequire as gf}from"node:module";var df=Object.create;var{getPrototypeOf:pf,defineProperty:q1,getOwnPropertyNames:lf}=Object;var sf=Object.prototype.hasOwnProperty;var FR=(R,f,q)=>{q=R!=null?df(pf(R)):{};let $=f||!R||!R.__esModule?q1(q,"default",{value:R,enumerable:!0}):q;for(let D of lf(R))if(!sf.call($,D))q1($,D,{get:()=>R[D],enumerable:!0});return $};var nf=(R,f)=>()=>(f||R((f={exports:{}}).exports,f),f.exports);var $1=(R,f)=>{for(var q in f)q1(R,q,{get:f[q],enumerable:!0,configurable:!0,set:($)=>f[q]=()=>$})};var H=(R,f)=>()=>(R&&(f=R(R=0)),f);var of=gf(import.meta.url);function LR(R,f){let q=process.env[R];if(!q)return f??null;return q}var a,RR,fR,N1;var QR=H(()=>{a=LR("JUDGMENT_API_KEY"),RR=LR("JUDGMENT_ORG_ID"),fR=LR("JUDGMENT_API_URL","https://api.judgmentlabs.ai"),N1=LR("JUDGMENT_LOG_LEVEL","warn")});class s{baseUrl;apiKey;organizationId;constructor(R,f,q){this.baseUrl=R,this.apiKey=f,this.organizationId=q}getBaseUrl(){return this.baseUrl}getApiKey(){return this.apiKey}getOrganizationId(){return this.organizationId}async request(R,f,q){let $=await fetch(f,{method:R,headers:{"Content-Type":"application/json",Authorization:`Bearer ${this.apiKey}`,"X-Organization-Id":this.organizationId},body:q!==void 0?JSON.stringify(q):void 0});if(!$.ok){let D=await $.text();throw Error(`HTTP ${$.status}: ${D}`)}return $.json()}async postOtelV1traces(){let R=this.baseUrl+"/otel/v1/traces";return this.request("POST",R,{})}async postOtelV1offlineTraces(){let R=this.baseUrl+"/otel/v1/offline-traces";return this.request("POST",R,{})}async postV1projectsResolve(R){let f=this.baseUrl+"/v1/projects/resolve/";return this.request("POST",f,R)}async postV1projects(R){let f=this.baseUrl+"/v1/projects";return this.request("POST",f,R)}async deleteV1projects(R){let f=this.baseUrl+`/v1/projects/${R}`;return this.request("DELETE",f,{})}async postV1projectsDatasets(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("POST",q,f)}async getV1projectsDatasets(R){let f=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("GET",f,void 0)}async getV1projectsDatasetsByDatasetIdentifierVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsDatasetsByDatasetIdentifierPage(R,f,q,$,D,P){let U=new URLSearchParams;if(q!==void 0)U.set("version",q);if($!==void 0)U.set("limit",$);if(D!==void 0)U.set("cursor_created_at",D);if(P!==void 0)U.set("cursor_example_id",P);let O=this.baseUrl+`/v1/projects/${R}/datasets/${f}/page`+(U.toString()?"?"+U.toString():"");return this.request("GET",O,void 0)}async postV1projectsDatasetsByDatasetIdentifierExamples(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/datasets/${f}/examples`;return this.request("POST",$,q)}async deleteV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("DELETE",q,{})}async getV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("GET",q,void 0)}async getV1projectsTestConfigs(R,f){let q=new URLSearchParams;if(f!==void 0)q.set("dataset_id",f);let $=this.baseUrl+`/v1/projects/${R}/test-configs`+(q.toString()?"?"+q.toString():"");return this.request("GET",$,void 0)}async postV1projectsTestConfigs(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs`;return this.request("POST",q,f)}async getV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestConfigsByTestConfigId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("PATCH",$,q)}async deleteV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("DELETE",q,{})}async getV1projectsTestRuns(R,f,q,$,D){let P=new URLSearchParams;if(f!==void 0)P.set("test_config_id",f);if(q!==void 0)P.set("dataset_id",q);if($!==void 0)P.set("status",$);if(D!==void 0)P.set("limit",D);let U=this.baseUrl+`/v1/projects/${R}/test-runs`+(P.toString()?"?"+P.toString():"");return this.request("GET",U,void 0)}async postV1projectsTestRuns(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs`;return this.request("POST",q,f)}async getV1projectsTestRunsByTestRunId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestRunsByTestRunIdStatus(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/status`;return this.request("PATCH",$,q)}async patchV1projectsTestRunsByTestRunIdSuccess(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/success`;return this.request("PATCH",$,q)}async getV1projectsTestRunsByTestRunIdItems(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/items`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdResults(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/results`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdGraph(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/graph`;return this.request("GET",q,void 0)}async getV1projectsExperimentsByRunId(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/experiments/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsEvalResults(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results`;return this.request("POST",q,f)}async postV1projectsEvalResultsExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/traces`;return this.request("POST",q,f)}async postV1projectsEvalQueueExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/examples`;return this.request("POST",q,f)}async postV1projectsEvalQueueTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/traces`;return this.request("POST",q,f)}async getV1projectsPromptsByName(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("commit_id",q);if($!==void 0)D.set("tag",$);let P=this.baseUrl+`/v1/projects/${R}/prompts/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsPrompts(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts`;return this.request("POST",q,f)}async postV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("POST",$,q)}async deleteV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("DELETE",$,q)}async getV1projectsPromptsByNameVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsScorers(R,f,q){let $=new URLSearchParams;if(f!==void 0)$.set("names",f);if(q!==void 0)$.set("is_trace",q);let D=this.baseUrl+`/v1/projects/${R}/scorers`+($.toString()?"?"+$.toString():"");return this.request("GET",D,void 0)}async getV1projectsScorersByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsScorersCustom(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom`;return this.request("POST",f,{})}async postV1projectsScorersCustomBundle(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom/bundle`;return this.request("POST",f,{})}async getV1projectsScorersCustomByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/custom/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsTracesByTraceIdTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/traces/${f}/tags`;return this.request("POST",$,q)}async getV1e2eFetchTraceByProjectNameByTraceId(R,f){let q=this.baseUrl+`/v1/e2e_fetch_trace/${R}/${f}`;return this.request("GET",q,void 0)}async getV1e2eTracesPerProject(R){let f=this.baseUrl+`/v1/e2e_traces_per_project/${R}`;return this.request("GET",f,void 0)}async postV1e2eFetchSpanScore(R){let f=this.baseUrl+"/v1/e2e_fetch_span_score/";return this.request("POST",f,R)}async postV1projectsJudges(R,f){let q=this.baseUrl+`/v1/projects/${R}/judges`;return this.request("POST",q,f)}async patchV1projectsJudgesByJudgeId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/judges/${f}`;return this.request("PATCH",$,q)}}var NR=()=>{};var F;var T=H(()=>{QR();F=class F{static RESET="\x1B[0m";static RED="\x1B[31m";static YELLOW="\x1B[33m";static GRAY="\x1B[90m";static Level={DEBUG:0,INFO:1,WARNING:2,ERROR:3,CRITICAL:4};static initialized=!1;static levelSetManually=!1;static currentLevel=F.Level.WARNING;static useColor=!0;static initialize(){if(!F.initialized){let R=process.env.JUDGMENT_NO_COLOR;if(F.useColor=!R&&process.stdout.isTTY,!F.levelSetManually){let f=N1.toLowerCase();if(f){let q={debug:F.Level.DEBUG,info:F.Level.INFO,warning:F.Level.WARNING,warn:F.Level.WARNING,error:F.Level.ERROR,critical:F.Level.CRITICAL};F.currentLevel=q[f]??F.Level.WARNING}}F.initialized=!0}}static setLevel(R){F.currentLevel=R,F.levelSetManually=!0}static setUseColor(R){F.useColor=R}static log(R,f){if(F.initialize(),R<F.currentLevel)return;let q=new Date().toISOString().replace("T"," ").substring(0,19),$=Object.keys(F.Level).find((U)=>F.Level[U]===R)??"UNKNOWN",D=`${q} - judgeval - ${$} - ${f}`;if(F.useColor)D=`${R===F.Level.DEBUG||R===F.Level.INFO?F.GRAY:R===F.Level.WARNING?F.YELLOW:F.RED}${D}${F.RESET}`;(R>=F.Level.ERROR?process.stderr:process.stdout).write(D+`
2
- `)}static debug(R){F.log(F.Level.DEBUG,R)}static info(R){F.log(F.Level.INFO,R)}static warning(R){F.log(F.Level.WARNING,R)}static warn(R){F.log(F.Level.WARNING,R)}static error(R){F.log(F.Level.ERROR,R)}static critical(R){F.log(F.Level.CRITICAL,R)}}});function AR(R){return new Promise((f)=>setTimeout(f,R))}async function A1(R,f={}){let{maxRetries:q=3,backoff:$=()=>1000,onRetry:D}=f;for(let P=1;P<=q;P++)try{return await R()}catch(U){if(P===q)throw U;D?.(P,U),await AR($(P))}throw Error("retry: exhausted all attempts")}var B1=()=>{};async function qR(R,f){let q=`org:${R.getOrganizationId()}:project:${f}`,$=v1.get(q);if($)return $;let D=D1.get(q);if(D)return D;let P=(async()=>{F.info(`Resolving project ID for project: ${f}`);let U=await A1(async()=>{let E=(await R.postV1projectsResolve({project_name:f})).project_id;if(!E)throw Error(`Project ID not found for project: ${f}`);return E},{maxRetries:3,backoff:(O)=>O*1000,onRetry:(O,E)=>{F.warning(`Failed to resolve project ID for '${f}' (attempt ${O}): ${String(E)}`)}});return F.info(`Resolved project ID: ${U}`),v1.set(q,U),U})();D1.set(q,P);try{return await P}finally{D1.delete(q)}}var v1,D1;var BR=H(()=>{T();B1();v1=new Map,D1=new Map});var WR=nf((S$,P1)=>{var IR=process||{},I1=IR.argv||[],vR=IR.env||{},tf=!(!!vR.NO_COLOR||I1.includes("--no-color"))&&(!!vR.FORCE_COLOR||I1.includes("--color")||IR.platform==="win32"||(IR.stdout||{}).isTTY&&vR.TERM!=="dumb"||!!vR.CI),cf=(R,f,q=R)=>($)=>{let D=""+$,P=D.indexOf(f,R.length);return~P?R+rf(D,f,q,P)+f:R+D+f},rf=(R,f,q,$)=>{let D="",P=0;do D+=R.substring(P,$)+q,P=$+f.length,$=R.indexOf(f,P);while(~$);return D+R.substring(P)},K1=(R=tf)=>{let f=R?cf:()=>String;return{isColorSupported:R,reset:f("\x1B[0m","\x1B[0m"),bold:f("\x1B[1m","\x1B[22m","\x1B[22m\x1B[1m"),dim:f("\x1B[2m","\x1B[22m","\x1B[22m\x1B[2m"),italic:f("\x1B[3m","\x1B[23m"),underline:f("\x1B[4m","\x1B[24m"),inverse:f("\x1B[7m","\x1B[27m"),hidden:f("\x1B[8m","\x1B[28m"),strikethrough:f("\x1B[9m","\x1B[29m"),black:f("\x1B[30m","\x1B[39m"),red:f("\x1B[31m","\x1B[39m"),green:f("\x1B[32m","\x1B[39m"),yellow:f("\x1B[33m","\x1B[39m"),blue:f("\x1B[34m","\x1B[39m"),magenta:f("\x1B[35m","\x1B[39m"),cyan:f("\x1B[36m","\x1B[39m"),white:f("\x1B[37m","\x1B[39m"),gray:f("\x1B[90m","\x1B[39m"),bgBlack:f("\x1B[40m","\x1B[49m"),bgRed:f("\x1B[41m","\x1B[49m"),bgGreen:f("\x1B[42m","\x1B[49m"),bgYellow:f("\x1B[43m","\x1B[49m"),bgBlue:f("\x1B[44m","\x1B[49m"),bgMagenta:f("\x1B[45m","\x1B[49m"),bgCyan:f("\x1B[46m","\x1B[49m"),bgWhite:f("\x1B[47m","\x1B[49m"),blackBright:f("\x1B[90m","\x1B[39m"),redBright:f("\x1B[91m","\x1B[39m"),greenBright:f("\x1B[92m","\x1B[39m"),yellowBright:f("\x1B[93m","\x1B[39m"),blueBright:f("\x1B[94m","\x1B[39m"),magentaBright:f("\x1B[95m","\x1B[39m"),cyanBright:f("\x1B[96m","\x1B[39m"),whiteBright:f("\x1B[97m","\x1B[39m"),bgBlackBright:f("\x1B[100m","\x1B[49m"),bgRedBright:f("\x1B[101m","\x1B[49m"),bgGreenBright:f("\x1B[102m","\x1B[49m"),bgYellowBright:f("\x1B[103m","\x1B[49m"),bgBlueBright:f("\x1B[104m","\x1B[49m"),bgMagentaBright:f("\x1B[105m","\x1B[49m"),bgCyanBright:f("\x1B[106m","\x1B[49m"),bgWhiteBright:f("\x1B[107m","\x1B[49m")}};P1.exports=K1();P1.exports.createColors=K1});var m;var SR=H(()=>{m=class m{exampleId;createdAt;name;_properties;constructor(R,f,q,$){this.exampleId=R,this.createdAt=f,this.name=q,this._properties=$}static create(R={}){return new m(crypto.randomUUID(),new Date().toISOString(),null,{...R})}static META_KEYS=new Set(["example_id","created_at","name","trace_id","span_id","offline_trace_id","agent_offline_trace_id"]);static from(R){let f={};for(let q of Object.keys(R))if(!m.META_KEYS.has(q))f[q]=R[q];return new m(R.example_id??"",R.created_at??"",R.name??null,f)}get(R){return this._properties[R]}has(R){return R in this._properties}get properties(){return{...this._properties}}toJSON(){let R={example_id:this.exampleId,created_at:this.createdAt,name:this.name};for(let[f,q]of Object.entries(this._properties))R[f]=q;return R}}});function qq(){let R=new WeakSet;return function(f,q){if(typeof q==="bigint")return q.toString();if(typeof q==="object"&&q!==null){if(R.has(q))return"[Circular]";R.add(q)}return q}}function w(R){try{let f=JSON.stringify(R);if(typeof f==="string")return f;return String(f)}catch{try{let f=JSON.stringify(R,qq());return typeof f==="string"?f:String(R)}catch(f){return F.error(`safeStringify failed: ${f}`),String(R)}}}function n(R,f){if(typeof R==="string"||typeof R==="number"||typeof R==="boolean")return R;return f(R)}var N=H(()=>{T()});var p1="1.2.0";var d1=()=>{};var lR;var V1=H(()=>{d1();lR=p1});var PR=()=>{};function S(R,f,q){try{return f()}catch($){let D=$ instanceof Error&&$.stack?`
3
- ${$.stack}`:"";return F.error(`[Caught] An exception was raised in ${R}: ${String($)}${D}`),q}}var k=H(()=>{T()});var l1=H(()=>{k()});function A(R,f={}){let{pre:q,post:$,error:D,finally:P}=f;return async function(...O){let E=q?S("immutableWrapAsync.pre",()=>q(...O)):void 0,Q;try{let Y=await R.apply(this,O);if($)Q=S("immutableWrapAsync.post",()=>$(E,Y,O));return Y}catch(Y){if(D)Q=S("immutableWrapAsync.error",()=>D(E,Y,O));throw Y}finally{if(P)S("immutableWrapAsync.finally",()=>{P(Q)})}}}var s1=H(()=>{k()});var n1=H(()=>{k()});function F1(R,f={}){let{pre:q,yield:$,post:D,error:P,finally:U}=f;return async function*(...E){let Q=q?S("immutableWrapAsyncIterator.pre",()=>q(...E)):void 0,Y;try{for await(let W of R(...E)){if($)S("immutableWrapAsyncIterator.yield",()=>{$(Q,W)});yield W}if(D)Y=S("immutableWrapAsyncIterator.post",()=>D(Q))}catch(W){if(P)Y=S("immutableWrapAsyncIterator.error",()=>P(Q,W));throw W}finally{if(U)S("immutableWrapAsyncIterator.finally",()=>{U(Y)})}}}var Q1=H(()=>{k()});function g(R,f){let q=R[Symbol.asyncIterator].bind(R),$=F1(()=>({[Symbol.asyncIterator]:q}),{yield:(D,P)=>{f.onYield(P)},post:()=>{f.onDone()},error:(D,P)=>{f.onError(P)},finally:()=>{f.onFinally()}});R[Symbol.asyncIterator]=()=>$()}var g1=H(()=>{Q1()});var ZR=H(()=>{l1();s1();n1();Q1();g1()});function XR(R,f){S("recordChatUsage",()=>{let q=f.prompt_tokens_details?.cached_tokens??0,$=f.prompt_tokens+f.completion_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.prompt_tokens-q:f.prompt_tokens,output_tokens:f.completion_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}var Y1=H(()=>{PR();K();k();N()});function i1(R){R.chat.completions.parse=A(R.chat.completions.parse.bind(R.chat.completions),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),q},post:(f,q)=>{if(!f)return;if(V.setOutput(w(q),f),q.usage)XR(f,q.usage);return V.recordLLMMetadata({model:q.model},f),f},error:(f,q)=>{if(f)V.setError(q,f);return f},finally:(f)=>{f?.end()}})}var o1=H(()=>{K();N();ZR();Y1()});function t1(R){R.chat.completions.create=A(R.chat.completions.create.bind(R.chat.completions),{pre:(f)=>{if(f.stream)f.stream_options??={include_usage:!0};let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O="";return g(U,{onYield(E){if(typeof E.choices[0]?.delta.content==="string")O+=E.choices[0].delta.content;if(E.usage)XR(D,E.usage)},onDone(){V.setOutput(O,D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)XR(D,P.usage);return V.recordLLMMetadata({model:P.model},D),f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var c1=H(()=>{K();N();ZR();Y1()});function r1(R,f){S("images.recordUsage",()=>{let q="input_tokens_details"in f?f.input_tokens_details:void 0,$=q?.image_tokens??0;if(V.recordLLMMetadata({non_cached_input_tokens:q?.text_tokens??0,output_tokens:f.output_tokens||void 0},R),$)V.setAttribute("judgment.usage.non_cached_input_image_tokens",$,R);if(f.output_tokens)V.setAttribute("judgment.usage.output_image_tokens",f.output_tokens,R);V.setAttribute("judgment.usage.metadata",w(f),R)})}function e1(R){R.images.generate=A(R.images.generate.bind(R.images),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O;return g(U,{onYield(E){if(Dq.has(E.type))O=E,r1(D,O.usage)},onDone(){V.setOutput(w(O??{}),D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)r1(D,P.usage);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Dq;var a1=H(()=>{PR();K();k();N();ZR();Dq=new Set(["image_generation.completed","image_edit.completed"])});function Rf(R,f){S("responses.recordUsage",()=>{let q=f.input_tokens_details.cached_tokens,$=f.input_tokens+f.output_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.input_tokens-q:f.input_tokens,output_tokens:f.output_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}function ff(R){R.responses.create=A(R.responses.create.bind(R.responses),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O="";return g(U,{onYield(E){if(E.type==="response.output_text.delta")O+=E.delta;if(E.type==="response.completed"){let Q=E.response;if(Q.usage)Rf(D,Q.usage);V.recordLLMMetadata({model:Q.model},D)}},onDone(){V.setOutput(O,D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)Rf(D,P.usage);if(typeof P.model==="string")V.recordLLMMetadata({model:P.model},D);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var qf=H(()=>{PR();K();k();N();ZR()});function W1(R){return S("wrapOpenAI",()=>{t1(R),i1(R),ff(R),e1(R)}),R}var $f=H(()=>{k();o1();c1();a1();qf()});function H1(R){return W1(R)}var S1=H(()=>{$f()});function Fq(R){return Function.prototype.toString.call(R)}function Qq(R){let f=Fq(R).replace(Vq,"");return f.match(Pq)||f.match(Uq)}function Df(R){let f=Qq(R);if(!f||!f[1])return[];return f[1].split(Eq).map((q)=>{return q.replace(Oq,(D,P,U)=>U).trim()}).filter((q)=>q.length>0)}var Pq,Uq,Eq,Oq,Vq;var Pf=H(()=>{Pq=/^([^(]+?)=>/,Uq=/^[^(]*\(\s*([^)]*)\)/m,Eq=/,/,Oq=/^\s*(_?)(\S+?)\1\s*$/,Vq=/((\/\/.*$)|(\/\*[\s\S]*?\*\/))/gm});import{INVALID_SPAN_CONTEXT as Yq,trace as Wq}from"@opentelemetry/api";class Uf{startSpan(){return Wq.wrapSpanContext(Yq)}startActiveSpan(R,...f){return(f.length===1?f[0]:f.length===2?f[1]:f[2])(this.startSpan())}}var nR;var C1=H(()=>{nR=new Uf});import{ROOT_CONTEXT as Hq,trace as Ef}from"@opentelemetry/api";function Vf(R){Of=R}function B(){return Of??Sq}var Sq,Of=null;var UR=H(()=>{T();C1();Sq={register(){},deregister(){},setActive(){return!1},getActiveTracer(){return null},getCurrentContext(){return Hq},setSpan(R,f){return Ef.setSpan(R,f)},wrapSpanContext(R){return Ef.wrapSpanContext(R)},getCurrentSpan(){return},getTracer(){return F.debug("No active tracer provider, returning NoOpTracer"),nR},addInstrumentation(){F.warning("No active tracer provider. Instrumentation was not registered.")},useSpan(R,f,q,$,D){return D()},attachContext(){},withContext(R,f){return f()},forceFlush(){return Promise.resolve()},shutdown(){return Promise.resolve()}}});var Ff="=",gR=";",JR=",",iR="baggage",Qf=180,Yf=4096,Wf=8192;import{baggageEntryMetadataFromString as Cq}from"@opentelemetry/api";function Hf(R){return R.reduce((f,q)=>{let $=`${f}${f!==""?JR:""}${q}`;return $.length>Wf?f:$},"")}function Sf(R){return R.getAllEntries().map(([f,q])=>{let $=`${encodeURIComponent(f)}=${encodeURIComponent(q.value)}`;if(q.metadata!==void 0)$+=gR+q.metadata.toString();return $})}function Cf(R){let f=R.split(gR);if(f.length<=0)return;let q=f.shift();if(!q)return;let $=q.indexOf(Ff);if($<=0)return;let D=decodeURIComponent(q.substring(0,$).trim()),P=decodeURIComponent(q.substring($+1).trim()),U;if(f.length>0)U=Cq(f.join(gR));return{key:D,value:P,metadata:U}}var Zf=()=>{};import{isTracingSuppressed as Zq}from"@opentelemetry/core";class i{inject(R,f,q){let $=u(R);if(!$||Zq(R))return;let D=Sf($).filter((U)=>U.length<=Yf).slice(0,Qf),P=Hf(D);if(P.length>0)q.set(f,iR,P)}extract(R,f,q){let $=q.get(f,iR),D=Array.isArray($)?$.join(JR):$;if(!D)return R;let P={};if(D.length===0)return R;if(D.split(JR).forEach((O)=>{let E=Cf(O);if(E){let Q={value:E.value};if(E.metadata)Q.metadata=E.metadata;P[E.key]=Q}}),Object.entries(P).length===0)return R;return hR(R,GR(P))}fields(){return[iR]}}var oR=H(()=>{wR();Zf()});var X1={};$1(X1,{setBaggage:()=>hR,getBaggage:()=>u,getActiveBaggage:()=>Jq,deleteBaggage:()=>Gq,createBaggage:()=>GR,baggageEntryMetadataFromString:()=>hq,JudgmentBaggagePropagator:()=>i});import{createContextKey as Xq,propagation as Xf}from"@opentelemetry/api";import{baggageEntryMetadataFromString as hq}from"@opentelemetry/api";function u(R){return R.getValue(Z1)}function Jq(){return u(B().getCurrentContext())}function hR(R,f){return R.setValue(Z1,f)}function Gq(R){return R.deleteValue(Z1)}var GR,Z1;var wR=H(()=>{UR();oR();GR=Xf.createBaggage.bind(Xf),Z1=Xq("baggage")});var h1={};$1(h1,{setGlobalTextmap:()=>Tq,inject:()=>zq,getGlobalTextmap:()=>J1,extract:()=>G1});import{defaultTextMapGetter as wq,defaultTextMapSetter as _q}from"@opentelemetry/api";import{CompositePropagator as bq,W3CTraceContextPropagator as Mq}from"@opentelemetry/core";function J1(){return Jf}function Tq(R){Jf=R}function Gf(R){if(R!==void 0)return R;return B().getCurrentContext()}function zq(R,f,q=_q){S("propagation.inject",()=>{J1().inject(Gf(f),R,q)})}function G1(R,f,q=wq){let $=Gf(f);return S("propagation.extract",()=>J1().extract($,R,q),$)}var Jf;var w1=H(()=>{k();oR();UR();Jf=new bq({propagators:[new Mq,new i]})});import{INVALID_SPAN_CONTEXT as xq,SpanStatusCode as ER}from"@opentelemetry/api";class V{projectName;projectId;apiKey;organizationId;apiUrl;environment;serializer;_tracerProvider;_client;_enableMonitoring;supportsLiveInstrumentation=!0;constructor(R,f,q,$,D,P,U,O,E,Q){this.projectName=R,this.projectId=f,this.apiKey=q,this.organizationId=$,this.apiUrl=D,this.environment=P,this.serializer=U,this._tracerProvider=O,this._client=E,this._enableMonitoring=Q}setActive(){return B().setActive(this)}static _getProxyProvider(){return B()}static _getSerializer(){return V._getProxyProvider().getActiveTracer()?.serializer??w}static _getCurrentTraceAndSpanId(){let f=V._getProxyProvider().getCurrentSpan();if(!f?.isRecording())return null;let q=f.spanContext();if(!q.traceId||!(q.traceFlags&1))return null;return[q.traceId,q.spanId]}static _emitPartial(){S("BaseTracer._emitPartial",()=>{let R=V._getProxyProvider().getActiveTracer();if(!R||!R.supportsLiveInstrumentation)return;R.getSpanProcessor().emitPartial()})}static getCurrentSpan(){return V._getProxyProvider().getCurrentSpan()}static async forceFlush(){await V._getProxyProvider().forceFlush()}static async shutdown(){await V._getProxyProvider().shutdown()}static registerOTELInstrumentation(R){S("BaseTracer.registerOTELInstrumentation",()=>{V._getProxyProvider().addInstrumentation(R)})}static wrap(R){return H1(R)}static getOTELTracer(){return V._getProxyProvider().getTracer(hf)}static startSpan(R,f){let q=V.getOTELTracer().startSpan(R,{attributes:f});return V._emitPartial(),q}static startActiveSpan(R,f){let{name:q,attributes:$}=R;return V.getOTELTracer().startActiveSpan(q,{attributes:$},(D)=>{V._emitPartial();try{let P=f(D);if(P instanceof Promise)return P.finally(()=>{D.end()});return D.end(),P}catch(P){throw D.end(),P}})}static span(R,f){return V.startActiveSpan({name:R},(q)=>{try{let $=f(q);if($ instanceof Promise)return $.catch((D)=>{throw q.setStatus({code:ER.ERROR,message:String(D)}),q.recordException(D),D});return $}catch($){throw q.setStatus({code:ER.ERROR,message:String($)}),q.recordException($),$}})}static with(R,f){return V.span(R,f)}static continueTrace(R,f){let q=V._getProxyProvider(),$=G1(R);return q.withContext($,()=>f($))}static observe(R,f){let q;if(typeof R==="function")q=R;else f=R;let{spanType:$="span",spanName:D,recordInput:P=!0,recordOutput:U=!0,fork:O=!1}=f??{},E=V._getProxyProvider(),Q=(Y)=>{let W=D??Y.name;return function(...C){let G=E.getTracer(hf);if(O&&E.getActiveTracer()!==null&&E.getCurrentSpan()?.isRecording()===!0){let X=V._getSerializer(),Z=G.startSpan(W),M=Z.spanContext();if($)Z.setAttribute("judgment.span_kind",$);let kR={"judgment.link.source_trace_id":M.traceId,"judgment.link.source_span_id":M.spanId};if($)kR["judgment.span_kind"]=$;let yR=E.setSpan(E.getCurrentContext(),E.wrapSpanContext(xq)),L=G.startSpan(W,{attributes:kR},yR),p=L.spanContext();Z.setAttribute("judgment.link.target_trace_id",p.traceId),Z.setAttribute("judgment.link.target_span_id",p.spanId);let e=()=>{L.end(),Z.end()},y1=(z)=>{for(let y of[L,Z])y.recordException(z),y.setStatus({code:ER.ERROR,message:String(z)})},L1=(z)=>{let y=n(z,X);L.setAttribute("judgment.output",y),Z.setAttribute("judgment.output",y)};if(P){let z=n(wf(Y,C),X);L.setAttribute("judgment.input",z),Z.setAttribute("judgment.input",z)}return V._emitPartial(),E.useSpan(L,!1,!1,!1,()=>{try{let z=Y.call(this,...C);if(z instanceof Promise)return z.then((y)=>{if(U)L1(y);return y}).catch((y)=>{throw y1(y),y}).finally(e);if(U)L1(z);return e(),z}catch(z){throw y1(z),e(),z}})}return G.startActiveSpan(W,(X)=>{if($)X.setAttribute("judgment.span_kind",$);try{if(P)X.setAttribute("judgment.input",n(wf(Y,C),V._getSerializer()));V._emitPartial();let Z=Y.call(this,...C);if(Z instanceof Promise)return Z.then((M)=>{if(U)X.setAttribute("judgment.output",n(M,V._getSerializer()));return M}).catch((M)=>{throw X.recordException(M),X.setStatus({code:ER.ERROR,message:String(M)}),M}).finally(()=>{X.end()});if(U)X.setAttribute("judgment.output",n(Z,V._getSerializer()));return X.end(),Z}catch(Z){throw X.recordException(Z),X.setStatus({code:ER.ERROR,message:String(Z)}),X.end(),Z}})}};if(!q)return Q;return Q(q)}static _resolveSpan(R){if(R)return R;return V._getProxyProvider().getCurrentSpan()}static setSpanKind(R,f){S("BaseTracer.setSpanKind",()=>{if(!R)return;let q=V._resolveSpan(f);if(q?.isRecording())q.setAttribute("judgment.span_kind",R)})}static setLLMSpan(){V.setSpanKind("llm")}static setToolSpan(){V.setSpanKind("tool")}static setGeneralSpan(){V.setSpanKind("span")}static setAttribute(R,f,q){S("BaseTracer.setAttribute",()=>{let $=V._resolveSpan(q);if(!$?.isRecording())return;if(!R||f==null)return;$.setAttribute(R,n(f,V._getSerializer()))})}static setAttributes(R,f){for(let[q,$]of Object.entries(R))if(f)V.setAttribute(q,$,f);else V.setAttribute(q,$)}static setInput(R,f){if(f)V.setAttribute("judgment.input",R,f);else V.setAttribute("judgment.input",R)}static setOutput(R,f){if(f)V.setAttribute("judgment.output",R,f);else V.setAttribute("judgment.output",R)}static setError(R,f){S("BaseTracer.setError",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;q.recordException(R),q.setStatus({code:ER.ERROR,message:String(R)})})}static recordLLMMetadata(R,f){S("BaseTracer.recordLLMMetadata",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;if(typeof R.model==="string")q.setAttribute("judgment.llm.model",R.model);if(typeof R.provider==="string")q.setAttribute("judgment.llm.provider",R.provider);if(typeof R.non_cached_input_tokens==="number")q.setAttribute("judgment.usage.non_cached_input_tokens",R.non_cached_input_tokens);if(typeof R.output_tokens==="number")q.setAttribute("judgment.usage.output_tokens",R.output_tokens);if(typeof R.cache_read_input_tokens==="number")q.setAttribute("judgment.usage.cache_read_input_tokens",R.cache_read_input_tokens);if(typeof R.cache_creation_input_tokens==="number")q.setAttribute("judgment.usage.cache_creation_input_tokens",R.cache_creation_input_tokens);if(typeof R.total_cost_usd==="number")q.setAttribute("judgment.usage.total_cost_usd",R.total_cost_usd)})}static _setPropagatingBaggageKey(R,f){S("BaseTracer._setPropagatingBaggageKey",()=>{let q=V._getProxyProvider(),$=q.getCurrentSpan();if(!$?.isRecording())return;$.setAttribute(R,f);let D=q.getCurrentContext(),P=(u(D)??GR()).setEntry(R,{value:f});q.attachContext(hR(D,P))})}static setCustomerId(R){V._setPropagatingBaggageKey("judgment.customer_id",R)}static setCustomerUserId(R){V._setPropagatingBaggageKey("judgment.customer_user_id",R)}static setSessionId(R){V._setPropagatingBaggageKey("judgment.session_id",R)}static tag(R){S("BaseTracer.tag",()=>{if(!R||Array.isArray(R)&&R.length===0)return;let q=V._getProxyProvider().getActiveTracer();if(!q?.projectId||!q._client)return;if(!q.supportsLiveInstrumentation)return;let $=V._getCurrentTraceAndSpanId();if(!$)return;let[D]=$,P=Array.isArray(R)?R:[R];q._client.postV1projectsTracesByTraceIdTags(q.projectId,D,{tags:P}).catch((U)=>{F.error(`tag failed: ${String(U)}`)})})}static asyncEvaluate(R,f){S("BaseTracer.asyncEvaluate",()=>{let{judge:q,example:$}=R,P=V._getProxyProvider().getActiveTracer();if(!P?.projectId)return;if(!P.supportsLiveInstrumentation)return;let U=V._resolveSpan(f);if(!U?.isRecording())return;let O=P.getSpanProcessor(),E=U.spanContext(),Q=O.stateIncr(E,"pending_evals_count"),Y={project_id:P.projectId,eval_name:`async_evaluate_${q}_${Q}`,judges:[{name:q}],examples:[{...$,example_id:crypto.randomUUID(),created_at:new Date().toISOString(),trace_id:E.traceId,span_id:E.spanId}],is_offline:!1,is_behavior:!1},W=O.stateAppend(E,"pending_evals",Y);U.setAttribute("judgment.pending_trace_eval",JSON.stringify(W))})}}function wf(R,f){try{let q=Df(R).map((D)=>D.replace(/^\.\.\./,"").split("=")[0].trim()).filter((D)=>D.length>0),$={};return q.forEach((D,P)=>{if(P<f.length)$[D]=f[P]}),$}catch{return{}}}var hf="judgeval";var K=H(()=>{PR();S1();Pf();k();T();N();wR();w1();UR()});import{ROOT_CONTEXT as mq,context as o}from"@opentelemetry/api";import{AsyncLocalStorage as bf}from"async_hooks";function b1(){return Mf.getStore()===!0}function Tf(R){if(_1=R,_f)return;let f=o;f.active=()=>{if(!b1())return kq();let q=tR.getStore();if(q)return q;return _1?_1():mq},f.with=(q,$,D,...P)=>{if(!b1())return yq(q,$,D,...P);return tR.run(q,()=>$.apply(D,P))},f.bind=(q,$)=>{if(!b1())return Lq(q,$);if(typeof $!=="function")return $;let D=$;return(...P)=>tR.run(q,()=>D(...P))},_f=!0}function M1(R,f){return Mf.run(!0,()=>tR.run(R,f))}var _f=!1,_1=null,Mf,tR,kq,yq,Lq;var zf=H(()=>{Mf=new bf,tR=new bf,kq=o.active.bind(o),yq=o.with.bind(o),Lq=o.bind.bind(o)});import{ROOT_CONTEXT as Nq,SpanStatusCode as jf,trace as OR}from"@opentelemetry/api";import{registerInstrumentations as Aq}from"@opentelemetry/instrumentation";import{AsyncLocalStorage as Bq}from"async_hooks";class xf{_provider;constructor(R){this._provider=R}startSpan(R,f,q){let $=q??this._provider.getCurrentContext();return this._provider._getDelegateTracer().startSpan(R,f,$)}startActiveSpan(R,...f){let q={},$=this._provider.getCurrentContext(),D;if(f.length===1)D=f[0];else if(f.length===2)q=f[0],D=f[1];else q=f[0],$=f[1],D=f[2];let P=this.startSpan(R,q,$);return this._provider.useSpan(P,!1,!1,!1,()=>D(P))}}class j{static _instance=null;_activeTracer=null;_instrumentations=[];_proxyTracer;_tracers=new Set;constructor(){this._proxyTracer=new xf(this),Vf(this),Tf(()=>this.getCurrentContext())}static getInstance(){return j._instance??=new j,j._instance}static installAsGlobalTracerProvider(){let R=j.getInstance();return OR.setGlobalTracerProvider(R)}register(R){this._tracers.add(R)}deregister(R){this._tracers.delete(R)}setActive(R){let f=this.getCurrentSpan();if(f?.isRecording()){if(OR.getSpan(this.getCurrentContext())===f)return F.error("Cannot set_active() while a root span is active. Keeping existing tracer provider."),!1}return this.register(R),this._activeTracer=R,!0}getActiveTracer(){return this._activeTracer}getCurrentContext(){return cR.getStore()??Nq}setSpan(R,f){return OR.setSpan(R,f)}wrapSpanContext(R){return OR.wrapSpanContext(R)}getCurrentSpan(){let R=this.getCurrentContext();return OR.getSpan(R)}hasActiveRootSpan(){if(!this.getCurrentSpan()?.isRecording())return!1;return!0}_getDelegateTracer(){let R=this._activeTracer;if(!R)return F.debug("No active tracer, returning NoOpTracer"),nR;return R._tracerProvider.getTracer(vq)}getTracer(R,f,q){return this._proxyTracer}addInstrumentation(R){try{Aq({tracerProvider:this,instrumentations:[R]}),this._instrumentations.push(R)}catch(f){F.error(`Failed to add instrumentation: ${String(f)}`)}}useSpan(R,f,q,$,D){let P=this.getCurrentContext(),U=OR.setSpan(P,R);return cR.run(U,()=>M1(U,()=>{try{let O=D();if(O instanceof Promise)return O.catch((E)=>{if(R.isRecording()){if(q)R.recordException(E);if($){let Q=E;R.setStatus({code:jf.ERROR,message:`${Q.name}: ${Q.message}`})}}throw E}).finally(()=>{if(f)R.end()});if(f)R.end();return O}catch(O){if(R.isRecording()){if(q)R.recordException(O);if($){let E=O;R.setStatus({code:jf.ERROR,message:`${E.name}: ${E.message}`})}}if(f)R.end();throw O}}))}attachContext(R){cR.enterWith(R)}withContext(R,f){return cR.run(R,()=>M1(R,f))}async forceFlush(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.forceFlush()));for(let f of R)if(f.status==="rejected")F.error(`forceFlush failed: ${String(f.reason)}`)}async shutdown(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.shutdown()));for(let f of R)if(f.status==="rejected")F.error(`shutdown failed: ${String(f.reason)}`);this._activeTracer=null,this._tracers.clear()}}var vq="judgeval",cR;var _R=H(()=>{T();zf();C1();UR();cR=new Bq});import{OTLPTraceExporter as Iq}from"@opentelemetry/exporter-trace-otlp-http";class v{_delegate;constructor(R,f,q,$){if(!R){this._delegate=null;return}this._delegate=new Iq({url:R,headers:{Authorization:`Bearer ${f}`,"X-Organization-Id":q,"X-Project-Id":$}})}export(R,f){F.info(`Exported ${R.length} spans`),this._delegate?.export(R,f)}shutdown(){return this._delegate?.shutdown()??Promise.resolve()}forceFlush(){return this._delegate?.forceFlush()??Promise.resolve()}}var bR=H(()=>{T()});import{ExportResultCode as Kq}from"@opentelemetry/core";var t;var rR=H(()=>{bR();t=class t extends v{constructor(){super("","","","")}export(R,f){f({code:Kq.SUCCESS})}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}}});class MR{_keyPredicate;constructor(R=T1){this._keyPredicate=R}onStart(R,f){let q=u(f)?.getAllEntries()??[];for(let[$,D]of q)if(this._keyPredicate($))R.setAttribute($,D.value)}onEnd(R){}forceFlush(){return Promise.resolve()}shutdown(){return Promise.resolve()}}var T1=()=>!0;var z1=H(()=>{wR()});import{BatchSpanProcessor as uq}from"@opentelemetry/sdk-trace-base";function VR(R){return`${R.traceId}:${R.spanId}`}function dq(R){return R[0]===0&&R[1]===0}var d;var TR=H(()=>{PR();k();UR();z1();d=class d extends uq{tracer;_state=new Map;_spanFinalizers;_baggageProcessor;constructor(R,f,q){super(f,q);this.tracer=R,this._spanFinalizers=new FinalizationRegistry(($)=>{this._cleanupSpanState($)}),this._baggageProcessor=new MR}_cleanupSpanState(R){this._state.delete(R)}_registerSpan(R){let f=R.spanContext();if(!f.traceId||!f.spanId)return;let q=VR(f);this._spanFinalizers.register(R,q)}stateSet(R,f,q){let $=VR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);D.set(f,q)}stateGet(R,f,q){let $=VR(R),D=this._state.get($);if(!D?.has(f))return q;return D.get(f)}stateIncr(R,f){let q=VR(R),$=this._state.get(q);if(!$)$=new Map,this._state.set(q,$);let D=$.get(f),P=typeof D==="number"?D:0;return $.set(f,P+1),P}stateAppend(R,f,q){let $=VR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);let P=D.get(f),U=Array.isArray(P)?[...P,q]:[q];return D.set(f,U),U}_emitSpan(R,f=!1){let q=R.spanContext();if(!q.traceId)return;let $=this.stateIncr(q,"judgment.update_id"),D={...R.attributes,"judgment.update_id":$};if(f)delete D["judgment.pending_trace_eval"];let P=Object.create(R);Object.defineProperty(P,"attributes",{value:D,writable:!1});let U=dq(R.endTime)?R.startTime:R.endTime;Object.defineProperty(P,"endTime",{value:U,writable:!1}),super.onEnd(P)}emitPartial(){S("JudgmentSpanProcessor.emitPartial",()=>{let R=B().getCurrentSpan();if(!R?.isRecording())return;let f=R.spanContext();if(!f.traceId)return;if(this.stateGet(f,"disable_partial_emit",!1))return;this._emitSpan(R,!0)})}onStart(R,f){S("JudgmentSpanProcessor.onStart",()=>{this._baggageProcessor.onStart(R,f),this._registerSpan(R)})}onEnd(R){S("JudgmentSpanProcessor.onEnd",()=>{let f=R.spanContext();if(!f.traceId){super.onEnd(R);return}let q=VR(f);try{if(!this.stateGet(f,"cancelled",!1))this._emitSpan(R)}finally{this._cleanupSpanState(q)}})}}});var zR;var j1=H(()=>{rR();TR();zR=class zR extends d{constructor(){super(null,new t)}onStart(R,f){}onEnd(R){}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}emitPartial(){}stateSet(R,f,q){}stateGet(R,f,q){return q}stateIncr(R,f){return 0}stateAppend(R,f,q){return[q]}}});import{defaultResource as pq,resourceFromAttributes as lq}from"@opentelemetry/resources";import{NodeTracerProvider as mf}from"@opentelemetry/sdk-trace-node";var I;var eR=H(()=>{QR();NR();T();BR();N();V1();K();_R();bR();rR();TR();j1();I=class I extends V{_spanExporter=null;_spanProcessor=null;constructor(R,f,q,$,D,P,U,O,E,Q){super(R,f,q,$,D,P,U,O,E,Q)}static async init(R={}){let f=R.apiKey??a,q=R.organizationId??RR,$=R.apiUrl??fR,D=R.projectName??null,P=R.serializer??w,U=!0;if(!D)F.warning("project_name not provided. Tracer will not export spans."),U=!1;if(!f)F.warning("api_key not provided. Tracer will not export spans."),U=!1;if(!q)F.warning("organization_id not provided. Tracer will not export spans."),U=!1;if(!$)F.warning("api_url not provided. Tracer will not export spans."),U=!1;let O=null,E=null;if(U&&D&&f&&q&&$){if(O=new s($,f,q),E=await qR(O,D).catch(()=>null),!E)F.warning(`Project '${D}' not found. Tracer will not export spans.`),U=!1}let Q={"service.name":D??"unknown","telemetry.sdk.name":"judgeval","telemetry.sdk.version":lR};if(R.environment)Q["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(Q,R.resourceAttributes);let Y=pq().merge(lq(Q)),W=new mf({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits}),C=new I(D,E,f,q,$,R.environment??null,P,W,O,U);if(U){let h=new mf({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[C.getSpanProcessor(),...R.spanProcessors??[]]});C._tracerProvider=h}if(j.getInstance().register(C),R.setActive??!0)C.setActive();return C}getSpanExporter(){if(this._spanExporter)return this._spanExporter;if(!this._enableMonitoring||!this.projectId||!this.apiKey||!this.organizationId||!this.apiUrl)this._spanExporter=new t;else{let R=this.apiUrl.endsWith("/")?this.apiUrl+"otel/v1/traces":this.apiUrl+"/otel/v1/traces";this._spanExporter=new v(R,this.apiKey,this.organizationId,this.projectId)}return this._spanExporter}getSpanProcessor(){if(this._spanProcessor)return this._spanProcessor;if(!this._enableMonitoring)this._spanProcessor=new zR;else this._spanProcessor=new d(this,this.getSpanExporter());return this._spanProcessor}}});var jR;var x1=H(()=>{SR();TR();jR=class jR extends d{_dataset;_exampleFields;_seenTraceIds=new Set;constructor(R,f,q){super(R,f);this._dataset=q.dataset,this._exampleFields={...q.exampleFields??{}}}_maybeCreateExample(R){if(R.parentSpanContext)return;let f=R.spanContext();if(!f?.traceId)return;if(this._seenTraceIds.has(f.traceId))return;this._seenTraceIds.add(f.traceId);let q=m.create({...this._exampleFields,offline_trace_id:f.traceId});this._dataset.push(q)}onEnd(R){try{this._maybeCreateExample(R)}finally{super.onEnd(R)}}}});var Lf={};$1(Lf,{OfflineTracer:()=>c});import{defaultResource as sq,resourceFromAttributes as nq}from"@opentelemetry/resources";import{NodeTracerProvider as kf}from"@opentelemetry/sdk-trace-node";var yf="otel/v1/offline-traces",c;var aR=H(()=>{QR();NR();BR();N();V1();_R();eR();bR();x1();c=class c extends I{supportsLiveInstrumentation=!1;_offlineApiUrl;_offlineApiKey;_offlineOrganizationId;_offlineProjectId;_dataset;_exampleFields;_offlineSpanExporter=null;_offlineSpanProcessor=null;constructor(R){super(R.projectName,R.projectId,R.apiKey,R.organizationId,R.apiUrl,R.environment,R.serializer,R.tracerProvider,R.client,!0);this._offlineApiUrl=R.apiUrl,this._offlineApiKey=R.apiKey,this._offlineOrganizationId=R.organizationId,this._offlineProjectId=R.projectId,this._dataset=R.dataset,this._exampleFields=R.exampleFields}static async create(R){let f=R.apiKey??a,q=R.organizationId??RR,$=R.apiUrl??fR,D=R.projectName,P=R.serializer??w;if(!D)throw Error("projectName is required for OfflineTracer");if(!f)throw Error("apiKey is required for OfflineTracer");if(!q)throw Error("organizationId is required for OfflineTracer");if(!$)throw Error("apiUrl is required for OfflineTracer");let U=new s($,f,q),O;try{O=await qR(U,D)}catch(G){throw Error(`Project '${D}' not found; cannot start OfflineTracer: ${String(G)}`)}let E={"service.name":D,"telemetry.sdk.name":"judgeval","telemetry.sdk.version":lR,"judgment.offline":"true"};if(R.environment)E["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(E,R.resourceAttributes);let Q=sq().merge(nq(E)),Y=new c({projectName:D,projectId:O,apiKey:f,organizationId:q,apiUrl:$,environment:R.environment??null,serializer:P,tracerProvider:new kf({resource:Q}),client:U,dataset:R.dataset,exampleFields:{...R.exampleFields??{}}}),W=new kf({resource:Q,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[Y.getSpanProcessor(),...R.spanProcessors??[]]});if(Y._tracerProvider=W,j.getInstance().register(Y),R.setActive??!0)Y.setActive();return Y}getSpanExporter(){if(this._offlineSpanExporter)return this._offlineSpanExporter;let R=this._offlineApiUrl.endsWith("/")?this._offlineApiUrl+yf:this._offlineApiUrl+"/"+yf;return this._offlineSpanExporter=new v(R,this._offlineApiKey,this._offlineOrganizationId,this._offlineProjectId),this._offlineSpanExporter}getSpanProcessor(){if(this._offlineSpanProcessor)return this._offlineSpanProcessor;return this._offlineSpanProcessor=new jR(this,this.getSpanExporter(),{dataset:this._dataset,exampleFields:this._exampleFields}),this._offlineSpanProcessor}}});QR();NR();BR();T();class YR{}T();var U1=FR(WR(),1);T();var J=FR(WR(),1),ef=2000;class $R{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async _poll(R,f,q,$){let D=Date.now();while(!0){let P=(Date.now()-D)/1000;if(P>$)throw Error(`Evaluation timed out after ${$}s`);let U=await this._client.getV1projectsExperimentsByRunId(R,f),O=U.results??[];if(O.length>=q){let Q=U.ui_results_url??"Failed to get UI results URL";return console.log(`${J.default.green("✓")} Evals completed and saved in ${J.default.bold(`${P.toFixed(1)}s`)}`),{results:O,url:Q}}await new Promise((Q)=>setTimeout(Q,ef))}}_displayResults(R,f,q,$){let D=[],P=0,U=0;console.log();for(let O=0;O<f.length;O++){let E=f[O],Q=E.scorers.every((Y)=>Boolean(Y.success));if(Q)P++,console.log(`${J.default.green("✓")} Example ${O+1}: ${J.default.green("PASSED")}`);else U++,console.log(`${J.default.red("✗")} Example ${O+1}: ${J.default.red("FAILED")}`);for(let Y of E.scorers){let W=Y.score!==null?Y.score.toFixed(3):"N/A",C=Y.success?J.default.green(W):J.default.red(W);console.log(` ${J.default.dim(`${Y.name}:`)} ${C} ${J.default.dim(`(threshold: ${Y.threshold})`)}`)}D.push({success:Q,scorers:E.scorers,example:R[O]})}if(console.log(),P===D.length)console.log(`${J.default.bold(J.default.green("✓ All tests passed!"))} (${P}/${D.length})`);else console.log(`${J.default.bold(J.default.yellow("⚠ Results:"))} ${J.default.green(`${P} passed`)} | ${J.default.red(`${U} failed`)}`);if(console.log(`${J.default.dim("View full details:")} ${J.default.underline(q)}`),console.log(),$&&D.some((O)=>!O.success)){let O=[`Evaluation failed: ${U}/${D.length} examples failed`];for(let E=0;E<D.length;E++)if(!D[E].success){O.push(` Example ${E+1}:`);for(let Q of D[E].scorers)if(!Q.success){if(O.push(` ${Q.name}: ${Q.score!==null?Q.score.toFixed(3):"N/A"} (threshold: ${Q.threshold})`),Q.reason)O.push(` ${Q.reason}`)}}throw Error(O.join(`
4
- `))}return D}async run(R,f,q,$=!1,D=300){if(!this._projectId)return F.error("Project ID is not resolved. Evaluation requires a valid project."),[];let P=this._projectId,U=crypto.randomUUID(),O=new Date().toISOString();console.log(),console.log(J.default.bold(J.default.cyan("Starting Evaluation"))),console.log(`${J.default.dim("Run:")} ${q}`),console.log(`${J.default.dim("Project:")} ${this._projectName}`),console.log(`${J.default.dim("Examples:")} ${R.length} | ${J.default.dim("Scorers:")} ${f.length}`),console.log();let E=this._buildPayload(U,P,q,O,R,f),Q=await this._submit(P,U,R,f,E),{results:Y,url:W}=await this._poll(P,U,Q,D);return this._displayResults(R,Y,W,$)}}class KR extends $R{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:[],custom_scorers:[]}}async _submit(R,f,q,$,D){let P=Date.now(),U=q.flatMap((W,C)=>$.map((G)=>G.score(W).then((h)=>({exampleIdx:C,scorer:G,result:h,error:null})).catch((h)=>({exampleIdx:C,scorer:G,result:null,error:String(h)})))),O=await Promise.all(U),E=((Date.now()-P)/1000).toFixed(1);console.log(`${U1.default.green("✓")} Scoring completed in ${U1.default.bold(`${E}s`)}`);let Q=new Map;for(let W of O){let C=Q.get(W.exampleIdx);if(!C)C=[],Q.set(W.exampleIdx,C);C.push(W)}let Y=q.map((W,C)=>{return{scorers_data:(Q.get(C)??[]).map((h)=>{if(h.error!==null)return{scorer_name:h.scorer.constructor.name,value:0,reason:"",error:h.error};let X=h.result;return{scorer_name:h.scorer.constructor.name,value:X.value,reason:X.reason,...X.citations&&{citations:X.citations.map((Z)=>({span_id:Z.spanId,span_attribute:Z.spanAttribute}))}}}),data_object:W.toJSON()}});return await this._client.postV1projectsEvalResultsExamples(R,{results:Y,run:D}),q.length}}var u1=FR(WR(),1);class uR extends $R{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:P.map((U)=>({name:U})),custom_scorers:[]}}async _submit(R,f,q,$,D){return await this._client.postV1projectsEvalQueueExamples(R,D),console.log(`${u1.default.green("✓")} Evaluation submitted`),q.length}}class HR{_local;_hosted;constructor(R,f,q){this._local=new KR(R,f,q),this._hosted=new uR(R,f,q)}run(R){let{examples:f,scorers:q,evalRunName:$,assertTest:D=!1,timeoutSeconds:P=300}=R,U=q.filter((E)=>typeof E==="string"),O=q.filter((E)=>E instanceof YR);if(O.length>0&&U.length>0)return F.error("Running both local and hosted scorers is not supported. Please run your evaluation with either local or hosted scorers, but not both."),Promise.resolve([]);if(O.length===0&&U.length===0)return F.error("No valid local or hosted scorers provided. Please provide at least one local or hosted scorer."),Promise.resolve([]);if(O.length>0)return this._local.run(f,O,$,D,P);return this._hosted.run(f,U,$,D,P)}}class dR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}create(){return new HR(this._client,this._projectId,this._projectName)}}SR();T();SR();class DR{name;projectId;projectName;datasetKind;examples;_client;constructor(R){this.name=R.name,this.projectId=R.projectId,this.projectName=R.projectName,this.datasetKind=R.datasetKind??"example",this.examples=R.examples??[],this._client=R.client??null}async addExamples(R,f=100){if(!this._client)return;for(let q=0;q<R.length;q+=f){let $=R.slice(q,q+f);await this._client.postV1projectsDatasetsByDatasetIdentifierExamples(this.projectId,this.name,{examples:$.map((D)=>D.toJSON())})}}async addFromJson(R,f=100){let{readFile:q}=await import("fs/promises"),$=await q(R,"utf-8"),P=JSON.parse($).map((U)=>{if(typeof U!=="object"||U===null)throw Error("Each item in the JSON array must be an object");return m.create(U)});await this.addExamples(P,f)}get length(){return this.examples.length}[Symbol.iterator](){return this.examples[Symbol.iterator]()}toString(){return`Dataset(name=${this.name}, examples=${this.examples.length})`}}var af="trace";function E1(R){if(typeof R!=="object"||R===null||Array.isArray(R))throw Error("Dataset schema must be a JSON object.");let f=R;if(f.type!=="object")throw Error('Dataset schema must declare top-level type "object".');let q=f.properties;if(typeof q!=="object"||q===null||Array.isArray(q))throw Error("Dataset schema must declare a 'properties' object.");let $=Object.entries(q).filter(([,D])=>typeof D==="object"&&D!==null&&D.type===af).map(([D])=>D);if($.length>1)throw Error(`A dataset may declare at most one trace column; found ${$.length}: ${$.join(", ")}.`)}function Rq(R){if(typeof R==="boolean")return"boolean";if(typeof R==="number")return Number.isInteger(R)?"integer":"number";if(typeof R==="string")return"string";if(Array.isArray(R))return"array";if(typeof R==="object"&&R!==null)return"object";return"string"}function O1(R){if(R.length===0)throw Error("Cannot infer a dataset schema without examples. Pass an explicit `schema` to client.datasets.create().");let f={},q=null;for(let $ of R){let D=new Set;for(let[P,U]of Object.entries($.properties)){if(U===null||U===void 0)continue;if(D.add(P),!(P in f))f[P]={type:Rq(U)}}if(q===null)q=D;else if(q.size!==D.size||![...q].every((P)=>D.has(P)))throw Error("All examples must share the same set of fields (dataset schemas require all declared properties). Pass an explicit `schema` or make all examples homogeneous.")}return{type:"object",properties:f}}class pR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async get(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsDatasetsByDatasetIdentifier(f,R),$=q.dataset_kind??"example",D=(q.examples??[]).map((P)=>m.from({...P.data??{},example_id:P.example_id,created_at:P.created_at??""}));return new DR({name:R,projectId:f,projectName:this._projectName,datasetKind:$,examples:D,client:this._client})}async create(R,f={}){let q=this._expectProjectId();if(!q)return null;let{examples:$=[],overwrite:D=!1,batchSize:P=100,schema:U}=f,O;if(U!==void 0)E1(U),O=U;else{if($.length===0)throw Error("Datasets require a JSON Schema. Pass `schema` to create(), or provide `examples` to infer one.");O=O1($)}await this._client.postV1projectsDatasets(q,{name:R,examples:[],dataset_kind:"example",overwrite:D,schema:O});let E=new DR({name:R,projectId:q,projectName:this._projectName,examples:$,client:this._client});if($.length>0)await E.addExamples($,P);return E}list(){let R=this._expectProjectId();if(!R)return Promise.resolve(null);return this._client.getV1projectsDatasets(R)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Dataset operations require a valid project."),null;return this._projectId}}T();class CR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async create(R){let f=this._expectProjectId();if(!f)return null;let q={name:R.name,prompt:R.prompt,model:R.model,score_type:R.scoreType};if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;return{judgeId:(await this._client.postV1projectsJudges(f,q)).judge_id,name:R.name,prompt:R.prompt,model:R.model,scoreType:R.scoreType,description:R.description??null,judgeDescription:R.judgeDescription??null,categories:R.categories??null,minScore:R.minScore??null,maxScore:R.maxScore??null,majorVersion:0,minorVersion:0}}async update(R){let f=this._expectProjectId();if(!f)return null;let q={};if(R.prompt!==void 0)q.prompt=R.prompt;if(R.model!==void 0)q.model=R.model;if(R.scoreType!==void 0)q.score_type=R.scoreType;if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;if(R.sourceMajorVersion!==void 0)q.source_major_version=R.sourceMajorVersion;if(R.sourceMinorVersion!==void 0)q.source_minor_version=R.sourceMinorVersion;if(R.targetMajorVersion!==void 0)q.target_major_version=R.targetMajorVersion;if(R.targetMinorVersion!==void 0)q.target_minor_version=R.targetMinorVersion;let $=await this._client.patchV1projectsJudgesByJudgeId(f,R.judgeId,q);return fq($)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Agent judge operations require a valid project."),null;return this._projectId}}function fq(R){let f=R.judge;return{judgeId:f.id,name:f.name,prompt:f.prompt??"",model:f.model??"",scoreType:f.score_type,description:f.description??null,judgeDescription:f.judge_description??null,categories:f.categories??null,minScore:f.min_score??null,maxScore:f.max_score??null,majorVersion:f.major_version??null,minorVersion:f.minor_version??null}}T();T();eR();aR();_R();function R1(R){let f=Array.isArray(R.judges)?R.judges.filter((q)=>typeof q==="object"&&q!==null):[];return{id:typeof R.id==="string"?R.id:"",name:typeof R.name==="string"?R.name:"",datasetId:typeof R.dataset_id==="string"?R.dataset_id:"",description:typeof R.description==="string"?R.description:null,createdAt:typeof R.created_at==="string"?R.created_at:null,judges:f}}function Nf(R){let f=[];for(let q of R)for(let $ of q.scorers)if($.success!==null&&$.success!==void 0)f.push($.success);if(f.length===0)return null;return f.every((q)=>q)}var _=FR(WR(),1);function x(R){return typeof R==="object"&&R!==null&&!Array.isArray(R)?R:{}}function r(R){return Array.isArray(R)?R:[]}function b(R){return typeof R==="string"?R:""}function Af(R){return R?"Yes":"No"}function gq(R){let{score_type:f,bool_value:q,str_value:$,num_value:D}=R;if(f==="binary")return typeof q==="boolean"?Af(q):null;if(f==="categorical")return typeof $==="string"?$:null;if(f==="numeric")return typeof D==="number"?D:null;if(typeof q==="boolean")return Af(q);if(typeof $==="string")return $;if(typeof D==="number")return D;return null}function iq(R){if(typeof R==="object"&&R!==null){let f=R.text;return typeof f==="string"&&f?f:null}if(typeof R==="string"){try{let f=JSON.parse(R);if(typeof f==="object"&&f!==null){let q=f.text;if(typeof q==="string")return q||null}}catch{}return R||null}return null}function Bf(R){if(!R||R.length===0)return;let f=[];for(let q of R){if(!q.name&&!q.judgeId)throw Error("judgeVersions entries require a 'name' (or 'judgeId') key");let $={};if(q.judgeId!==void 0)$.judge_id=q.judgeId;if(q.name!==void 0)$.name=q.name;if(q.tag!==void 0)$.tag=q.tag;if(q.version!==void 0)$.version=q.version;if(q.majorVersion!==void 0)$.major_version=q.majorVersion;if(q.minorVersion!==void 0)$.minor_version=q.minorVersion;f.push($)}return f}function vf(R,f,q){let $=[];for(let D of R){let P=b(D.example_id),U=x(D.data),O=[];for(let E of r(D.scorers)){let Q=x(E),Y={judge_id:Q.judge_id,judge_major_version:Q.judge_major_version,judge_minor_version:Q.judge_minor_version},W=iq(Q.reason);if(W)Y.reason=W;O.push({name:b(Q.judge_name),value:gq(Q),scoreType:typeof Q.score_type==="string"?Q.score_type:null,error:typeof Q.error==="string"?Q.error:null,success:typeof Q.success==="boolean"?Q.success:null,metadata:Y})}if(q){let E=Boolean(q({...U},O));for(let Q of O)Q.success=E}$.push({exampleId:P,data:U,scorers:O,agentOfflineTraceId:f[P]})}return $}function If(R){if(R.status!=="completed")throw Error(`Test run ${R.testRunId} finished with status '${R.status}'`);let f=R.results.filter((q)=>q.scorers.some(($)=>$.success===!1)).map((q)=>q.exampleId);if(f.length>0||R.passed!==!0)throw Error(`Test run ${R.testRunId} failed its pass condition for ${f.length} example(s): ${JSON.stringify(f)}`)}function Kf(R,f,q){console.log(),console.log(_.default.bold(_.default.cyan("Starting Offline Test"))),console.log(`${_.default.dim("Config:")} ${R}`),console.log(`${_.default.dim("Project:")} ${f}`),console.log(`${_.default.dim("Examples:")} ${q}`),console.log()}function uf(R,f){if(console.log(),R.forEach((q,$)=>{if(q.scorers.some((P)=>P.success!==null&&P.success!==void 0)){let P=q.scorers.every((U)=>U.success===!0);console.log(P?`${_.default.green("✓")} Example ${$+1}: ${_.default.green("PASSED")}`:`${_.default.red("✗")} Example ${$+1}: ${_.default.red("FAILED")}`)}else console.log(`${_.default.cyan("•")} Example ${$+1}:`);for(let P of q.scorers){if(P.error){console.log(` ${_.default.dim(`${P.name}:`)} ${_.default.red(`error: ${P.error}`)}`);continue}let U=P.value===null?"N/A":String(P.value),O=P.success===!0?_.default.green(U):P.success===!1?_.default.red(U):U;console.log(` ${_.default.dim(`${P.name}:`)} ${O}`)}}),console.log(),f)console.log(`${_.default.dim("View full details:")} ${_.default.underline(f)}`),console.log()}var oq=new Set(["completed","error","cancelled"]),tq=100,cq=200,rq=2000,f1=1e4;class xR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async resolveDatasetVersion(R,f){let q=await this._client.getV1projectsDatasetsByDatasetIdentifierVersions(this._projectId,R.datasetId),$=r(q.versions).filter((D)=>typeof D==="object"&&D!==null).map((D)=>D);if(f===void 0){if($.length===0)throw Error(`Dataset of test config '${R.name}' has no versions`);return $.reduce((D,P)=>Number(P.version_number??0)>Number(D.version_number??0)?P:D)}if(typeof f==="number"){let D=$.find((P)=>Number(P.version_number??0)===f);if(D)return D}else{let D=$.find((P)=>P.version_id===f);if(D)return D}throw Error(`Dataset version ${JSON.stringify(f)} does not exist for the dataset of test config '${R.name}'`)}async fetchExamples(R,f){let q=[],$,D,P=0;for(;;){if(P>=f1)throw Error(`fetchExamples exceeded ${f1} pages for dataset of test config '${R.name}'`);let U=await this._client.getV1projectsDatasetsByDatasetIdentifierPage(this._projectId,R.datasetId,String(f),String(tq),$,D);P+=1;for(let Q of r(U.entries)){let Y=x(x(Q).example),W=Y.data;if(typeof W==="string")try{W=JSON.parse(W)}catch{W={}}q.push({exampleId:b(Y.example_id),data:x(W),offlineTraceId:typeof Y.offline_trace_id==="string"?Y.offline_trace_id:null,createdAt:typeof Y.created_at==="string"?Y.created_at:null})}let O=x(U.metadata),E=x(O.nextCursor);if(!O.hasMore||Object.keys(E).length===0)break;$=b(E.created_at),D=b(E.example_id)}return q}createTestRun(R,f={}){let q={test_config_id:R.id,source:f.source??"sdk"};if(f.name)q.name=f.name;if(typeof f.datasetVersion==="number")q.dataset_version_number=f.datasetVersion;else if(typeof f.datasetVersion==="string")q.dataset_version_id=f.datasetVersion;let $=Bf(f.judgeVersions);if($)q.judge_versions=$;let D=f.agentTraces??{};if(Object.keys(D).length>0)q.agent_traces=Object.entries(D).map(([P,U])=>({example_id:P,agent_offline_trace_id:U}));return this._client.postV1projectsTestRuns(this._projectId,q)}async runAgent(R,f){let q=[],$=j.getInstance().getActiveTracer();await c.create({projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl(),dataset:q,setActive:!0});let D={};try{let P=I.observe(R,{spanType:"agent"});for(let U of f){let O=q.length;try{await P(U.data)}catch(E){F.error(`Agent entrypoint raised for example ${U.exampleId}: ${String(E)}`)}for(let E of q.slice(O)){let Q=E.get("offline_trace_id");if(U.exampleId&&typeof Q==="string"){D[U.exampleId]=Q;break}}}}finally{if(await I.forceFlush(),$)$.setActive()}return D}async waitForCompletion(R,f){let q=Date.now();for(;;){if((Date.now()-q)/1000>f)throw Error(`Test run ${R} did not complete within ${f}s`);let $=await this._client.getV1projectsTestRunsByTestRunId(this._projectId,R),D=b(x($.test_run).status);if(oq.has(D))return D;await AR(rq)}}async fetchItems(R){let f=[],q="",$,D=0;for(;;){if(D>=f1)throw Error(`fetchItems exceeded ${f1} pages for test run ${R}`);let P=await this._client.getV1projectsTestRunsByTestRunIdItems(this._projectId,R,String(cq),$);D+=1;for(let O of r(P.results))f.push(x(O));if(!q)q=b(P.ui_results_url);let U=P.next_cursor;if(!P.has_more||typeof U!=="string"||!U)break;$=U}return{items:f,uiResultsUrl:q}}async reportSuccess(R,f,q,$){let D=new Map,P=new Map,U=(Y,W,C,G)=>`${Y}\x00${W}\x00${C}\x00${G}`,O=(Y,W)=>`${Y}\x00${W}`;for(let Y of r(f.evaluation_runs)){let W=x(Y),C=W.run_id;if(typeof C!=="string"||!C)continue;D.set(U(b(W.example_id),b(W.judge_id),Number(W.judge_major_version??0),Number(W.judge_minor_version??0)),C),P.set(O(b(W.example_id),b(W.judge_name)),C)}let E=new Map;for(let Y of $)E.set(Y.exampleId,Y);let Q=[];for(let Y of q){let W=b(Y.example_id),C=E.get(W),G=C?C.scorers.map((X)=>X.success??null):[],h=r(Y.scorers);for(let X=0;X<h.length;X+=1){let Z=x(h[X]),M=D.get(U(W,b(Z.judge_id),Number(Z.judge_major_version??0),Number(Z.judge_minor_version??0)))??P.get(O(W,b(Z.judge_name)));if(!M){F.warning(`No evaluation run ref for scorer ${b(Z.judge_name)} of example ${W}; skipping its success update`);continue}Q.push({evaluation_run_id:M,success:X<G.length?G[X]:null})}}if(Q.length===0)return;await this._client.patchV1projectsTestRunsByTestRunIdSuccess(this._projectId,R,{successes:Q})}async run(R,f={}){let{agentFunction:q,judgeVersions:$,datasetVersion:D,passConditionFn:P,assertTest:U=!1,timeoutSeconds:O=600,runName:E}=f;if(U&&!P)throw Error("assertTest=true requires a passConditionFn to decide whether each row passes.");let Q=await this.resolveDatasetVersion(R,D),Y=Number(Q.version_number??0),W=await this.fetchExamples(R,Y);Kf(R.name,this._projectName,W.length),F.info(`Dataset version ${Y}: ${W.length} example(s)`);let C=typeof D==="string"?D:Y,G={};if(q&&W.length>0)G=await this.runAgent(q,W);let h=await this.createTestRun(R,{datasetVersion:C,judgeVersions:$,agentTraces:G,name:E}),X=x(h.test_run),Z=b(X.id),M=b(h.ui_results_url);F.info(`Created test run ${Z} over ${W.length} examples`);let kR=await this.waitForCompletion(Z,O),{items:yR,uiResultsUrl:L}=await this.fetchItems(Z);if(L)M=L;let p=vf(yR,G,P);if(P)await this.reportSuccess(Z,h,yR,p);uf(p,M||void 0);let e={testRunId:Z,status:kR,uiResultsUrl:M||void 0,results:p,agentOfflineTraceIds:G,passed:Nf(p)};if(U)If(e);return e}}var eq=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function m1(R){return eq.test(R)}class mR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async createConfig(R,f,q,$){let D=this._expectProjectId();if(!D)return null;let P=[];for(let E of q)if(typeof E==="string")P.push(m1(E)?{judge_id:E}:{name:E});else if(E.judgeId)P.push({judge_id:E.judgeId});else if(E.name)P.push({name:E.name});else throw Error("judges entries must be judge names or objects with a 'judgeId' or 'name'");let U={name:R,judges:P};if($!==void 0)U.description=$;if(m1(f))U.dataset_id=f;else U.dataset_name=f;let O=await this._client.postV1projectsTestConfigs(D,U);return F.info(`Created test config ${R}`),R1(O.test_config??{})}async getConfig(R){let f=this._expectProjectId();if(!f)return null;if(m1(R)){let D=await this._client.getV1projectsTestConfigsByTestConfigId(f,R);return R1(D.test_config??{})}let $=(await this.listConfigs()??[]).find((D)=>D.name===R);if(!$)return F.error(`Test config '${R}' not found`),null;return $}async listConfigs(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsTestConfigs(f,R);return(Array.isArray(q.test_configs)?q.test_configs:[]).map((D)=>R1(D))}async deleteConfig(R){let f=this._expectProjectId();if(!f)return!1;return await this._client.deleteV1projectsTestConfigsByTestConfigId(f,R),F.info(`Deleted test config ${R}`),!0}async run(R,f={}){let q=this._expectProjectId();if(!q)return null;let $=typeof R==="string"?await this.getConfig(R):R;if(!$||!$.id)return null;return new xR(this._client,q,this._projectName).run($,f)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Offline-test operations require a valid project."),null;return this._projectId}}class k1{_client;_projectName;_projectId;constructor(R,f,q){this._client=R,this._projectName=f,this._projectId=q}static async create(R){let f=R.apiKey??a,q=R.organizationId??RR,$=R.apiUrl??fR;if(!f)throw Error("API key is required");if(!q)throw Error("Organization ID is required");if(!$)throw Error("API URL is required");if(!R.projectName)throw Error("Project name is required");let D=new s($,f,q),P=null;try{P=await qR(D,R.projectName)}catch{F.warning(`Project '${R.projectName}' not found. Some operations requiring project_id will be skipped.`)}return new k1(D,R.projectName,P)}async offlineTracer(R){let{OfflineTracer:f}=await Promise.resolve().then(() => (aR(),Lf));return f.create({...R,projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl()})}get datasets(){return new pR(this._client,this._projectId,this._projectName)}get evaluation(){return new dR(this._client,this._projectId,this._projectName)}get agentJudges(){return new CR(this._client,this._projectId,this._projectName)}get offlineTests(){return new mR(this._client,this._projectId,this._projectName)}}K();bR();rR();_R();TR();j1();x1();z1();oR();wR();w1();eR();aR();S1();SR();export{W1 as wrapOpenAI,H1 as wrap,E1 as validateDatasetSchema,h1 as propagation,O1 as inferSchemaFromExamples,X1 as baggage,I as Tracer,c as OfflineTracer,mR as OfflineTestsFactory,xR as OfflineTestRunner,jR as OfflineJudgmentSpanProcessor,zR as NoOpSpanProcessor,t as NoOpSpanExporter,j as JudgmentTracerProvider,v as JudgmentSpanExporter,MR as JudgmentBaggageSpanProcessor,i as JudgmentBaggagePropagator,k1 as Judgeval,YR as Judge,m as Example,HR as Evaluation,DR as Dataset,V as BaseTracer,CR as AgentJudgeFactory,T1 as ALLOW_ALL_BAGGAGE_KEYS};
1
+ import{createRequire as gf}from"node:module";var df=Object.create;var{getPrototypeOf:pf,defineProperty:q1,getOwnPropertyNames:lf}=Object;var sf=Object.prototype.hasOwnProperty;var FR=(R,f,q)=>{q=R!=null?df(pf(R)):{};let $=f||!R||!R.__esModule?q1(q,"default",{value:R,enumerable:!0}):q;for(let D of lf(R))if(!sf.call($,D))q1($,D,{get:()=>R[D],enumerable:!0});return $};var nf=(R,f)=>()=>(f||R((f={exports:{}}).exports,f),f.exports);var $1=(R,f)=>{for(var q in f)q1(R,q,{get:f[q],enumerable:!0,configurable:!0,set:($)=>f[q]=()=>$})};var H=(R,f)=>()=>(R&&(f=R(R=0)),f);var of=gf(import.meta.url);function NR(R,f){let q=process.env[R];if(!q)return f??null;return q}var a,RR,fR,y1;var QR=H(()=>{a=NR("JUDGMENT_API_KEY"),RR=NR("JUDGMENT_ORG_ID"),fR=NR("JUDGMENT_API_URL","https://api.judgmentlabs.ai"),y1=NR("JUDGMENT_LOG_LEVEL","warn")});class s{baseUrl;apiKey;organizationId;constructor(R,f,q){this.baseUrl=R,this.apiKey=f,this.organizationId=q}getBaseUrl(){return this.baseUrl}getApiKey(){return this.apiKey}getOrganizationId(){return this.organizationId}async request(R,f,q){let $=await fetch(f,{method:R,headers:{"Content-Type":"application/json",Authorization:`Bearer ${this.apiKey}`,"X-Organization-Id":this.organizationId},body:q!==void 0?JSON.stringify(q):void 0});if(!$.ok){let D=await $.text();throw Error(`HTTP ${$.status}: ${D}`)}return $.json()}async postOtelV1traces(){let R=this.baseUrl+"/otel/v1/traces";return this.request("POST",R,{})}async postOtelV1offlineTraces(){let R=this.baseUrl+"/otel/v1/offline-traces";return this.request("POST",R,{})}async postV1projectsResolve(R){let f=this.baseUrl+"/v1/projects/resolve/";return this.request("POST",f,R)}async postV1projects(R){let f=this.baseUrl+"/v1/projects";return this.request("POST",f,R)}async deleteV1projects(R){let f=this.baseUrl+`/v1/projects/${R}`;return this.request("DELETE",f,{})}async postV1projectsDatasets(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("POST",q,f)}async getV1projectsDatasets(R){let f=this.baseUrl+`/v1/projects/${R}/datasets`;return this.request("GET",f,void 0)}async getV1projectsDatasetsByDatasetIdentifierVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsDatasetsByDatasetIdentifierPage(R,f,q,$,D,P){let U=new URLSearchParams;if(q!==void 0)U.set("version",q);if($!==void 0)U.set("limit",$);if(D!==void 0)U.set("cursor_created_at",D);if(P!==void 0)U.set("cursor_example_id",P);let O=this.baseUrl+`/v1/projects/${R}/datasets/${f}/page`+(U.toString()?"?"+U.toString():"");return this.request("GET",O,void 0)}async postV1projectsDatasetsByDatasetIdentifierExamples(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/datasets/${f}/examples`;return this.request("POST",$,q)}async deleteV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("DELETE",q,{})}async getV1projectsDatasetsByDatasetIdentifier(R,f){let q=this.baseUrl+`/v1/projects/${R}/datasets/${f}`;return this.request("GET",q,void 0)}async getV1projectsTestConfigs(R,f){let q=new URLSearchParams;if(f!==void 0)q.set("dataset_id",f);let $=this.baseUrl+`/v1/projects/${R}/test-configs`+(q.toString()?"?"+q.toString():"");return this.request("GET",$,void 0)}async postV1projectsTestConfigs(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs`;return this.request("POST",q,f)}async getV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestConfigsByTestConfigId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("PATCH",$,q)}async deleteV1projectsTestConfigsByTestConfigId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-configs/${f}`;return this.request("DELETE",q,{})}async getV1projectsTestRuns(R,f,q,$,D){let P=new URLSearchParams;if(f!==void 0)P.set("test_config_id",f);if(q!==void 0)P.set("dataset_id",q);if($!==void 0)P.set("status",$);if(D!==void 0)P.set("limit",D);let U=this.baseUrl+`/v1/projects/${R}/test-runs`+(P.toString()?"?"+P.toString():"");return this.request("GET",U,void 0)}async postV1projectsTestRuns(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs`;return this.request("POST",q,f)}async getV1projectsTestRunsByTestRunId(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}`;return this.request("GET",q,void 0)}async patchV1projectsTestRunsByTestRunIdStatus(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/status`;return this.request("PATCH",$,q)}async patchV1projectsTestRunsByTestRunIdSuccess(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/success`;return this.request("PATCH",$,q)}async getV1projectsTestRunsByTestRunIdItems(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/items`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdResults(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/results`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async getV1projectsTestRunsByTestRunIdGraph(R,f){let q=this.baseUrl+`/v1/projects/${R}/test-runs/${f}/graph`;return this.request("GET",q,void 0)}async getV1projectsExperimentsByRunId(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("limit",q);if($!==void 0)D.set("cursor",$);let P=this.baseUrl+`/v1/projects/${R}/experiments/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsEvalResults(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results`;return this.request("POST",q,f)}async postV1projectsEvalResultsExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-results/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/examples`;return this.request("POST",q,f)}async postV1projectsEvaluateTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/evaluate/traces`;return this.request("POST",q,f)}async postV1projectsEvalQueueExamples(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/examples`;return this.request("POST",q,f)}async postV1projectsEvalQueueTraces(R,f){let q=this.baseUrl+`/v1/projects/${R}/eval-queue/traces`;return this.request("POST",q,f)}async getV1projectsPromptsByName(R,f,q,$){let D=new URLSearchParams;if(q!==void 0)D.set("commit_id",q);if($!==void 0)D.set("tag",$);let P=this.baseUrl+`/v1/projects/${R}/prompts/${f}`+(D.toString()?"?"+D.toString():"");return this.request("GET",P,void 0)}async postV1projectsPrompts(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts`;return this.request("POST",q,f)}async postV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("POST",$,q)}async deleteV1projectsPromptsByNameTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/prompts/${f}/tags`;return this.request("DELETE",$,q)}async getV1projectsPromptsByNameVersions(R,f){let q=this.baseUrl+`/v1/projects/${R}/prompts/${f}/versions`;return this.request("GET",q,void 0)}async getV1projectsScorers(R,f,q){let $=new URLSearchParams;if(f!==void 0)$.set("names",f);if(q!==void 0)$.set("is_trace",q);let D=this.baseUrl+`/v1/projects/${R}/scorers`+($.toString()?"?"+$.toString():"");return this.request("GET",D,void 0)}async getV1projectsScorersByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsScorersCustom(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom`;return this.request("POST",f,{})}async postV1projectsScorersCustomBundle(R){let f=this.baseUrl+`/v1/projects/${R}/scorers/custom/bundle`;return this.request("POST",f,{})}async getV1projectsScorersCustomByNameExists(R,f){let q=this.baseUrl+`/v1/projects/${R}/scorers/custom/${f}/exists`;return this.request("GET",q,void 0)}async postV1projectsTracesByTraceIdTags(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/traces/${f}/tags`;return this.request("POST",$,q)}async getV1e2eFetchTraceByProjectNameByTraceId(R,f){let q=this.baseUrl+`/v1/e2e_fetch_trace/${R}/${f}`;return this.request("GET",q,void 0)}async getV1e2eTracesPerProject(R){let f=this.baseUrl+`/v1/e2e_traces_per_project/${R}`;return this.request("GET",f,void 0)}async postV1e2eFetchSpanScore(R){let f=this.baseUrl+"/v1/e2e_fetch_span_score/";return this.request("POST",f,R)}async postV1projectsJudges(R,f){let q=this.baseUrl+`/v1/projects/${R}/judges`;return this.request("POST",q,f)}async patchV1projectsJudgesByJudgeId(R,f,q){let $=this.baseUrl+`/v1/projects/${R}/judges/${f}`;return this.request("PATCH",$,q)}}var yR=()=>{};var F;var T=H(()=>{QR();F=class F{static RESET="\x1B[0m";static RED="\x1B[31m";static YELLOW="\x1B[33m";static GRAY="\x1B[90m";static Level={DEBUG:0,INFO:1,WARNING:2,ERROR:3,CRITICAL:4};static initialized=!1;static levelSetManually=!1;static currentLevel=F.Level.WARNING;static useColor=!0;static initialize(){if(!F.initialized){let R=process.env.JUDGMENT_NO_COLOR;if(F.useColor=!R&&process.stdout.isTTY,!F.levelSetManually){let f=y1.toLowerCase();if(f){let q={debug:F.Level.DEBUG,info:F.Level.INFO,warning:F.Level.WARNING,warn:F.Level.WARNING,error:F.Level.ERROR,critical:F.Level.CRITICAL};F.currentLevel=q[f]??F.Level.WARNING}}F.initialized=!0}}static setLevel(R){F.currentLevel=R,F.levelSetManually=!0}static setUseColor(R){F.useColor=R}static log(R,f){if(F.initialize(),R<F.currentLevel)return;let q=new Date().toISOString().replace("T"," ").substring(0,19),$=Object.keys(F.Level).find((U)=>F.Level[U]===R)??"UNKNOWN",D=`${q} - judgeval - ${$} - ${f}`;if(F.useColor)D=`${R===F.Level.DEBUG||R===F.Level.INFO?F.GRAY:R===F.Level.WARNING?F.YELLOW:F.RED}${D}${F.RESET}`;(R>=F.Level.ERROR?process.stderr:process.stdout).write(D+`
2
+ `)}static debug(R){F.log(F.Level.DEBUG,R)}static info(R){F.log(F.Level.INFO,R)}static warning(R){F.log(F.Level.WARNING,R)}static warn(R){F.log(F.Level.WARNING,R)}static error(R){F.log(F.Level.ERROR,R)}static critical(R){F.log(F.Level.CRITICAL,R)}}});function AR(R){return new Promise((f)=>setTimeout(f,R))}async function A1(R,f={}){let{maxRetries:q=3,backoff:$=()=>1000,onRetry:D}=f;for(let P=1;P<=q;P++)try{return await R()}catch(U){if(P===q)throw U;D?.(P,U),await AR($(P))}throw Error("retry: exhausted all attempts")}var B1=()=>{};async function qR(R,f){let q=`org:${R.getOrganizationId()}:project:${f}`,$=v1.get(q);if($)return $;let D=D1.get(q);if(D)return D;let P=(async()=>{F.info(`Resolving project ID for project: ${f}`);let U=await A1(async()=>{let E=(await R.postV1projectsResolve({project_name:f})).project_id;if(!E)throw Error(`Project ID not found for project: ${f}`);return E},{maxRetries:3,backoff:(O)=>O*1000,onRetry:(O,E)=>{F.warning(`Failed to resolve project ID for '${f}' (attempt ${O}): ${String(E)}`)}});return F.info(`Resolved project ID: ${U}`),v1.set(q,U),U})();D1.set(q,P);try{return await P}finally{D1.delete(q)}}var v1,D1;var BR=H(()=>{T();B1();v1=new Map,D1=new Map});var WR=nf((S$,P1)=>{var IR=process||{},I1=IR.argv||[],vR=IR.env||{},tf=!(!!vR.NO_COLOR||I1.includes("--no-color"))&&(!!vR.FORCE_COLOR||I1.includes("--color")||IR.platform==="win32"||(IR.stdout||{}).isTTY&&vR.TERM!=="dumb"||!!vR.CI),cf=(R,f,q=R)=>($)=>{let D=""+$,P=D.indexOf(f,R.length);return~P?R+rf(D,f,q,P)+f:R+D+f},rf=(R,f,q,$)=>{let D="",P=0;do D+=R.substring(P,$)+q,P=$+f.length,$=R.indexOf(f,P);while(~$);return D+R.substring(P)},K1=(R=tf)=>{let f=R?cf:()=>String;return{isColorSupported:R,reset:f("\x1B[0m","\x1B[0m"),bold:f("\x1B[1m","\x1B[22m","\x1B[22m\x1B[1m"),dim:f("\x1B[2m","\x1B[22m","\x1B[22m\x1B[2m"),italic:f("\x1B[3m","\x1B[23m"),underline:f("\x1B[4m","\x1B[24m"),inverse:f("\x1B[7m","\x1B[27m"),hidden:f("\x1B[8m","\x1B[28m"),strikethrough:f("\x1B[9m","\x1B[29m"),black:f("\x1B[30m","\x1B[39m"),red:f("\x1B[31m","\x1B[39m"),green:f("\x1B[32m","\x1B[39m"),yellow:f("\x1B[33m","\x1B[39m"),blue:f("\x1B[34m","\x1B[39m"),magenta:f("\x1B[35m","\x1B[39m"),cyan:f("\x1B[36m","\x1B[39m"),white:f("\x1B[37m","\x1B[39m"),gray:f("\x1B[90m","\x1B[39m"),bgBlack:f("\x1B[40m","\x1B[49m"),bgRed:f("\x1B[41m","\x1B[49m"),bgGreen:f("\x1B[42m","\x1B[49m"),bgYellow:f("\x1B[43m","\x1B[49m"),bgBlue:f("\x1B[44m","\x1B[49m"),bgMagenta:f("\x1B[45m","\x1B[49m"),bgCyan:f("\x1B[46m","\x1B[49m"),bgWhite:f("\x1B[47m","\x1B[49m"),blackBright:f("\x1B[90m","\x1B[39m"),redBright:f("\x1B[91m","\x1B[39m"),greenBright:f("\x1B[92m","\x1B[39m"),yellowBright:f("\x1B[93m","\x1B[39m"),blueBright:f("\x1B[94m","\x1B[39m"),magentaBright:f("\x1B[95m","\x1B[39m"),cyanBright:f("\x1B[96m","\x1B[39m"),whiteBright:f("\x1B[97m","\x1B[39m"),bgBlackBright:f("\x1B[100m","\x1B[49m"),bgRedBright:f("\x1B[101m","\x1B[49m"),bgGreenBright:f("\x1B[102m","\x1B[49m"),bgYellowBright:f("\x1B[103m","\x1B[49m"),bgBlueBright:f("\x1B[104m","\x1B[49m"),bgMagentaBright:f("\x1B[105m","\x1B[49m"),bgCyanBright:f("\x1B[106m","\x1B[49m"),bgWhiteBright:f("\x1B[107m","\x1B[49m")}};P1.exports=K1();P1.exports.createColors=K1});var m;var SR=H(()=>{m=class m{exampleId;createdAt;name;_properties;constructor(R,f,q,$){this.exampleId=R,this.createdAt=f,this.name=q,this._properties=$}static create(R={}){return new m(crypto.randomUUID(),new Date().toISOString(),null,{...R})}static META_KEYS=new Set(["example_id","created_at","name","trace_id","span_id","offline_trace_id","agent_offline_trace_id"]);static from(R){let f={};for(let q of Object.keys(R))if(!m.META_KEYS.has(q))f[q]=R[q];return new m(R.example_id??"",R.created_at??"",R.name??null,f)}get(R){return this._properties[R]}has(R){return R in this._properties}get properties(){return{...this._properties}}toJSON(){let R={example_id:this.exampleId,created_at:this.createdAt,name:this.name};for(let[f,q]of Object.entries(this._properties))R[f]=q;return R}}});function qq(){let R=new WeakSet;return function(f,q){if(typeof q==="bigint")return q.toString();if(typeof q==="object"&&q!==null){if(R.has(q))return"[Circular]";R.add(q)}return q}}function w(R){try{let f=JSON.stringify(R);if(typeof f==="string")return f;return String(f)}catch{try{let f=JSON.stringify(R,qq());return typeof f==="string"?f:String(R)}catch(f){return F.error(`safeStringify failed: ${f}`),String(R)}}}function n(R,f){if(typeof R==="string"||typeof R==="number"||typeof R==="boolean")return R;return f(R)}var y=H(()=>{T()});var p1="1.2.1";var d1=()=>{};var lR;var V1=H(()=>{d1();lR=p1});var PR=()=>{};function S(R,f,q){try{return f()}catch($){let D=$ instanceof Error&&$.stack?`
3
+ ${$.stack}`:"";return F.error(`[Caught] An exception was raised in ${R}: ${String($)}${D}`),q}}var k=H(()=>{T()});var l1=H(()=>{k()});function A(R,f={}){let{pre:q,post:$,error:D,finally:P}=f;return async function(...O){let E=q?S("immutableWrapAsync.pre",()=>q(...O)):void 0,Q;try{let Y=await R.apply(this,O);if($)Q=S("immutableWrapAsync.post",()=>$(E,Y,O));return Y}catch(Y){if(D)Q=S("immutableWrapAsync.error",()=>D(E,Y,O));throw Y}finally{if(P)S("immutableWrapAsync.finally",()=>{P(Q)})}}}var s1=H(()=>{k()});var n1=H(()=>{k()});function F1(R,f={}){let{pre:q,yield:$,post:D,error:P,finally:U}=f;return async function*(...E){let Q=q?S("immutableWrapAsyncIterator.pre",()=>q(...E)):void 0,Y;try{for await(let W of R(...E)){if($)S("immutableWrapAsyncIterator.yield",()=>{$(Q,W)});yield W}if(D)Y=S("immutableWrapAsyncIterator.post",()=>D(Q))}catch(W){if(P)Y=S("immutableWrapAsyncIterator.error",()=>P(Q,W));throw W}finally{if(U)S("immutableWrapAsyncIterator.finally",()=>{U(Y)})}}}var Q1=H(()=>{k()});function g(R,f){let q=R[Symbol.asyncIterator].bind(R),$=F1(()=>({[Symbol.asyncIterator]:q}),{yield:(D,P)=>{f.onYield(P)},post:()=>{f.onDone()},error:(D,P)=>{f.onError(P)},finally:()=>{f.onFinally()}});R[Symbol.asyncIterator]=()=>$()}var g1=H(()=>{Q1()});var ZR=H(()=>{l1();s1();n1();Q1();g1()});function XR(R,f){S("recordChatUsage",()=>{let q=f.prompt_tokens_details?.cached_tokens??0,$=f.prompt_tokens+f.completion_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.prompt_tokens-q:f.prompt_tokens,output_tokens:f.completion_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}var Y1=H(()=>{PR();K();k();y()});function i1(R){R.chat.completions.parse=A(R.chat.completions.parse.bind(R.chat.completions),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),q},post:(f,q)=>{if(!f)return;if(V.setOutput(w(q),f),q.usage)XR(f,q.usage);return V.recordLLMMetadata({model:q.model},f),f},error:(f,q)=>{if(f)V.setError(q,f);return f},finally:(f)=>{f?.end()}})}var o1=H(()=>{K();y();ZR();Y1()});function t1(R){R.chat.completions.create=A(R.chat.completions.create.bind(R.chat.completions),{pre:(f)=>{if(f.stream)f.stream_options??={include_usage:!0};let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O="";return g(U,{onYield(E){if(typeof E.choices[0]?.delta.content==="string")O+=E.choices[0].delta.content;if(E.usage)XR(D,E.usage)},onDone(){V.setOutput(O,D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)XR(D,P.usage);return V.recordLLMMetadata({model:P.model},D),f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var c1=H(()=>{K();y();ZR();Y1()});function r1(R,f){S("images.recordUsage",()=>{let q="input_tokens_details"in f?f.input_tokens_details:void 0,$=q?.image_tokens??0;if(V.recordLLMMetadata({non_cached_input_tokens:q?.text_tokens??0,output_tokens:f.output_tokens||void 0},R),$)V.setAttribute("judgment.usage.non_cached_input_image_tokens",$,R);if(f.output_tokens)V.setAttribute("judgment.usage.output_image_tokens",f.output_tokens,R);V.setAttribute("judgment.usage.metadata",w(f),R)})}function e1(R){R.images.generate=A(R.images.generate.bind(R.images),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O;return g(U,{onYield(E){if(Dq.has(E.type))O=E,r1(D,O.usage)},onDone(){V.setOutput(w(O??{}),D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)r1(D,P.usage);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var Dq;var a1=H(()=>{PR();K();k();y();ZR();Dq=new Set(["image_generation.completed","image_edit.completed"])});function Rf(R,f){S("responses.recordUsage",()=>{let q=f.input_tokens_details.cached_tokens,$=f.input_tokens+f.output_tokens+q;V.recordLLMMetadata({non_cached_input_tokens:$>f.total_tokens?f.input_tokens-q:f.input_tokens,output_tokens:f.output_tokens||void 0,cache_read_input_tokens:q||void 0},R),V.setAttribute("judgment.usage.metadata",w(f),R)})}function ff(R){R.responses.create=A(R.responses.create.bind(R.responses),{pre:(f)=>{let q=V.startSpan("OPENAI_API_CALL");return V.setSpanKind("llm",q),V.recordLLMMetadata({model:f.model},q),V.setInput(f,q),{span:q,proxied:!1}},post:(f,q,$)=>{if(!f)return;let{span:D}=f;if($[0].stream){let U=q,O="";return g(U,{onYield(E){if(E.type==="response.output_text.delta")O+=E.delta;if(E.type==="response.completed"){let Q=E.response;if(Q.usage)Rf(D,Q.usage);V.recordLLMMetadata({model:Q.model},D)}},onDone(){V.setOutput(O,D)},onError(E){V.setError(E,D)},onFinally(){D.end()}}),{span:D,proxied:!0}}let P=q;if(V.setOutput(w(P),D),P.usage)Rf(D,P.usage);if(typeof P.model==="string")V.recordLLMMetadata({model:P.model},D);return f},error:(f,q)=>{if(f)V.setError(q,f.span);return f},finally:(f)=>{if(f&&!f.proxied)f.span.end()}})}var qf=H(()=>{PR();K();k();y();ZR()});function W1(R){return S("wrapOpenAI",()=>{t1(R),i1(R),ff(R),e1(R)}),R}var $f=H(()=>{k();o1();c1();a1();qf()});function H1(R){return W1(R)}var S1=H(()=>{$f()});function Fq(R){return Function.prototype.toString.call(R)}function Qq(R){let f=Fq(R).replace(Vq,"");return f.match(Pq)||f.match(Uq)}function Df(R){let f=Qq(R);if(!f||!f[1])return[];return f[1].split(Eq).map((q)=>{return q.replace(Oq,(D,P,U)=>U).trim()}).filter((q)=>q.length>0)}var Pq,Uq,Eq,Oq,Vq;var Pf=H(()=>{Pq=/^([^(]+?)=>/,Uq=/^[^(]*\(\s*([^)]*)\)/m,Eq=/,/,Oq=/^\s*(_?)(\S+?)\1\s*$/,Vq=/((\/\/.*$)|(\/\*[\s\S]*?\*\/))/gm});import{INVALID_SPAN_CONTEXT as Yq,trace as Wq}from"@opentelemetry/api";class Uf{startSpan(){return Wq.wrapSpanContext(Yq)}startActiveSpan(R,...f){return(f.length===1?f[0]:f.length===2?f[1]:f[2])(this.startSpan())}}var nR;var C1=H(()=>{nR=new Uf});import{ROOT_CONTEXT as Hq,trace as Ef}from"@opentelemetry/api";function Vf(R){Of=R}function B(){return Of??Sq}var Sq,Of=null;var UR=H(()=>{T();C1();Sq={register(){},deregister(){},setActive(){return!1},getActiveTracer(){return null},getCurrentContext(){return Hq},setSpan(R,f){return Ef.setSpan(R,f)},wrapSpanContext(R){return Ef.wrapSpanContext(R)},getCurrentSpan(){return},getTracer(){return F.debug("No active tracer provider, returning NoOpTracer"),nR},addInstrumentation(){F.warning("No active tracer provider. Instrumentation was not registered.")},useSpan(R,f,q,$,D){return D()},attachContext(){},withContext(R,f){return f()},forceFlush(){return Promise.resolve()},shutdown(){return Promise.resolve()}}});var Ff="=",gR=";",JR=",",iR="baggage",Qf=180,Yf=4096,Wf=8192;import{baggageEntryMetadataFromString as Cq}from"@opentelemetry/api";function Hf(R){return R.reduce((f,q)=>{let $=`${f}${f!==""?JR:""}${q}`;return $.length>Wf?f:$},"")}function Sf(R){return R.getAllEntries().map(([f,q])=>{let $=`${encodeURIComponent(f)}=${encodeURIComponent(q.value)}`;if(q.metadata!==void 0)$+=gR+q.metadata.toString();return $})}function Cf(R){let f=R.split(gR);if(f.length<=0)return;let q=f.shift();if(!q)return;let $=q.indexOf(Ff);if($<=0)return;let D=decodeURIComponent(q.substring(0,$).trim()),P=decodeURIComponent(q.substring($+1).trim()),U;if(f.length>0)U=Cq(f.join(gR));return{key:D,value:P,metadata:U}}var Zf=()=>{};import{isTracingSuppressed as Zq}from"@opentelemetry/core";class i{inject(R,f,q){let $=u(R);if(!$||Zq(R))return;let D=Sf($).filter((U)=>U.length<=Yf).slice(0,Qf),P=Hf(D);if(P.length>0)q.set(f,iR,P)}extract(R,f,q){let $=q.get(f,iR),D=Array.isArray($)?$.join(JR):$;if(!D)return R;let P={};if(D.length===0)return R;if(D.split(JR).forEach((O)=>{let E=Cf(O);if(E){let Q={value:E.value};if(E.metadata)Q.metadata=E.metadata;P[E.key]=Q}}),Object.entries(P).length===0)return R;return hR(R,GR(P))}fields(){return[iR]}}var oR=H(()=>{wR();Zf()});var X1={};$1(X1,{setBaggage:()=>hR,getBaggage:()=>u,getActiveBaggage:()=>Jq,deleteBaggage:()=>Gq,createBaggage:()=>GR,baggageEntryMetadataFromString:()=>hq,JudgmentBaggagePropagator:()=>i});import{createContextKey as Xq,propagation as Xf}from"@opentelemetry/api";import{baggageEntryMetadataFromString as hq}from"@opentelemetry/api";function u(R){return R.getValue(Z1)}function Jq(){return u(B().getCurrentContext())}function hR(R,f){return R.setValue(Z1,f)}function Gq(R){return R.deleteValue(Z1)}var GR,Z1;var wR=H(()=>{UR();oR();GR=Xf.createBaggage.bind(Xf),Z1=Xq("baggage")});var h1={};$1(h1,{setGlobalTextmap:()=>Tq,inject:()=>zq,getGlobalTextmap:()=>J1,extract:()=>G1});import{defaultTextMapGetter as wq,defaultTextMapSetter as _q}from"@opentelemetry/api";import{CompositePropagator as bq,W3CTraceContextPropagator as Mq}from"@opentelemetry/core";function J1(){return Jf}function Tq(R){Jf=R}function Gf(R){if(R!==void 0)return R;return B().getCurrentContext()}function zq(R,f,q=_q){S("propagation.inject",()=>{J1().inject(Gf(f),R,q)})}function G1(R,f,q=wq){let $=Gf(f);return S("propagation.extract",()=>J1().extract($,R,q),$)}var Jf;var w1=H(()=>{k();oR();UR();Jf=new bq({propagators:[new Mq,new i]})});import{INVALID_SPAN_CONTEXT as xq,SpanStatusCode as ER}from"@opentelemetry/api";class V{projectName;projectId;apiKey;organizationId;apiUrl;environment;serializer;_tracerProvider;_client;_enableMonitoring;supportsLiveInstrumentation=!0;constructor(R,f,q,$,D,P,U,O,E,Q){this.projectName=R,this.projectId=f,this.apiKey=q,this.organizationId=$,this.apiUrl=D,this.environment=P,this.serializer=U,this._tracerProvider=O,this._client=E,this._enableMonitoring=Q}setActive(){return B().setActive(this)}static _getProxyProvider(){return B()}static _getSerializer(){return V._getProxyProvider().getActiveTracer()?.serializer??w}static _getCurrentTraceAndSpanId(){let f=V._getProxyProvider().getCurrentSpan();if(!f?.isRecording())return null;let q=f.spanContext();if(!q.traceId||!(q.traceFlags&1))return null;return[q.traceId,q.spanId]}static _emitPartial(){S("BaseTracer._emitPartial",()=>{let R=V._getProxyProvider().getActiveTracer();if(!R||!R.supportsLiveInstrumentation)return;R.getSpanProcessor().emitPartial()})}static getCurrentSpan(){return V._getProxyProvider().getCurrentSpan()}static async forceFlush(){await V._getProxyProvider().forceFlush()}static async shutdown(){await V._getProxyProvider().shutdown()}static registerOTELInstrumentation(R){S("BaseTracer.registerOTELInstrumentation",()=>{V._getProxyProvider().addInstrumentation(R)})}static wrap(R){return H1(R)}static getOTELTracer(){return V._getProxyProvider().getTracer(hf)}static startSpan(R,f){let q=V.getOTELTracer().startSpan(R,{attributes:f});return V._emitPartial(),q}static startActiveSpan(R,f){let{name:q,attributes:$}=R;return V.getOTELTracer().startActiveSpan(q,{attributes:$},(D)=>{V._emitPartial();try{let P=f(D);if(P instanceof Promise)return P.finally(()=>{D.end()});return D.end(),P}catch(P){throw D.end(),P}})}static span(R,f){return V.startActiveSpan({name:R},(q)=>{try{let $=f(q);if($ instanceof Promise)return $.catch((D)=>{throw q.setStatus({code:ER.ERROR,message:String(D)}),q.recordException(D),D});return $}catch($){throw q.setStatus({code:ER.ERROR,message:String($)}),q.recordException($),$}})}static with(R,f){return V.span(R,f)}static continueTrace(R,f){let q=V._getProxyProvider(),$=G1(R);return q.withContext($,()=>f($))}static observe(R,f){let q;if(typeof R==="function")q=R;else f=R;let{spanType:$="span",spanName:D,recordInput:P=!0,recordOutput:U=!0,fork:O=!1}=f??{},E=V._getProxyProvider(),Q=(Y)=>{let W=D??Y.name;return function(...C){let G=E.getTracer(hf);if(O&&E.getActiveTracer()!==null&&E.getCurrentSpan()?.isRecording()===!0){let X=V._getSerializer(),Z=G.startSpan(W),M=Z.spanContext();if($)Z.setAttribute("judgment.span_kind",$);let kR={"judgment.link.source_trace_id":M.traceId,"judgment.link.source_span_id":M.spanId};if($)kR["judgment.span_kind"]=$;let LR=E.setSpan(E.getCurrentContext(),E.wrapSpanContext(xq)),N=G.startSpan(W,{attributes:kR},LR),p=N.spanContext();Z.setAttribute("judgment.link.target_trace_id",p.traceId),Z.setAttribute("judgment.link.target_span_id",p.spanId);let e=()=>{N.end(),Z.end()},L1=(z)=>{for(let L of[N,Z])L.recordException(z),L.setStatus({code:ER.ERROR,message:String(z)})},N1=(z)=>{let L=n(z,X);N.setAttribute("judgment.output",L),Z.setAttribute("judgment.output",L)};if(P){let z=n(wf(Y,C),X);N.setAttribute("judgment.input",z),Z.setAttribute("judgment.input",z)}return V._emitPartial(),E.useSpan(N,!1,!1,!1,()=>{try{let z=Y.call(this,...C);if(z instanceof Promise)return z.then((L)=>{if(U)N1(L);return L}).catch((L)=>{throw L1(L),L}).finally(e);if(U)N1(z);return e(),z}catch(z){throw L1(z),e(),z}})}return G.startActiveSpan(W,(X)=>{if($)X.setAttribute("judgment.span_kind",$);try{if(P)X.setAttribute("judgment.input",n(wf(Y,C),V._getSerializer()));V._emitPartial();let Z=Y.call(this,...C);if(Z instanceof Promise)return Z.then((M)=>{if(U)X.setAttribute("judgment.output",n(M,V._getSerializer()));return M}).catch((M)=>{throw X.recordException(M),X.setStatus({code:ER.ERROR,message:String(M)}),M}).finally(()=>{X.end()});if(U)X.setAttribute("judgment.output",n(Z,V._getSerializer()));return X.end(),Z}catch(Z){throw X.recordException(Z),X.setStatus({code:ER.ERROR,message:String(Z)}),X.end(),Z}})}};if(!q)return Q;return Q(q)}static _resolveSpan(R){if(R)return R;return V._getProxyProvider().getCurrentSpan()}static setSpanKind(R,f){S("BaseTracer.setSpanKind",()=>{if(!R)return;let q=V._resolveSpan(f);if(q?.isRecording())q.setAttribute("judgment.span_kind",R)})}static setLLMSpan(){V.setSpanKind("llm")}static setToolSpan(){V.setSpanKind("tool")}static setGeneralSpan(){V.setSpanKind("span")}static setAttribute(R,f,q){S("BaseTracer.setAttribute",()=>{let $=V._resolveSpan(q);if(!$?.isRecording())return;if(!R||f==null)return;$.setAttribute(R,n(f,V._getSerializer()))})}static setAttributes(R,f){for(let[q,$]of Object.entries(R))if(f)V.setAttribute(q,$,f);else V.setAttribute(q,$)}static setInput(R,f){if(f)V.setAttribute("judgment.input",R,f);else V.setAttribute("judgment.input",R)}static setOutput(R,f){if(f)V.setAttribute("judgment.output",R,f);else V.setAttribute("judgment.output",R)}static setError(R,f){S("BaseTracer.setError",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;q.recordException(R),q.setStatus({code:ER.ERROR,message:String(R)})})}static recordLLMMetadata(R,f){S("BaseTracer.recordLLMMetadata",()=>{let q=V._resolveSpan(f);if(!q?.isRecording())return;if(typeof R.model==="string")q.setAttribute("judgment.llm.model",R.model);if(typeof R.provider==="string")q.setAttribute("judgment.llm.provider",R.provider);if(typeof R.non_cached_input_tokens==="number")q.setAttribute("judgment.usage.non_cached_input_tokens",R.non_cached_input_tokens);if(typeof R.output_tokens==="number")q.setAttribute("judgment.usage.output_tokens",R.output_tokens);if(typeof R.cache_read_input_tokens==="number")q.setAttribute("judgment.usage.cache_read_input_tokens",R.cache_read_input_tokens);if(typeof R.cache_creation_input_tokens==="number")q.setAttribute("judgment.usage.cache_creation_input_tokens",R.cache_creation_input_tokens);if(typeof R.total_cost_usd==="number")q.setAttribute("judgment.usage.total_cost_usd",R.total_cost_usd)})}static _setPropagatingBaggageKey(R,f){S("BaseTracer._setPropagatingBaggageKey",()=>{let q=V._getProxyProvider(),$=q.getCurrentSpan();if(!$?.isRecording())return;$.setAttribute(R,f);let D=q.getCurrentContext(),P=(u(D)??GR()).setEntry(R,{value:f});q.attachContext(hR(D,P))})}static setCustomerId(R){V._setPropagatingBaggageKey("judgment.customer_id",R)}static setCustomerUserId(R){V._setPropagatingBaggageKey("judgment.customer_user_id",R)}static setSessionId(R){V._setPropagatingBaggageKey("judgment.session_id",R)}static setPropagatingAttribute(R,f){if(R.startsWith("judgment.")){F.error(`setPropagatingAttribute: key "${R}" uses the reserved "judgment." prefix; ignoring`);return}V._setPropagatingBaggageKey(R,f)}static tag(R){S("BaseTracer.tag",()=>{if(!R||Array.isArray(R)&&R.length===0)return;let q=V._getProxyProvider().getActiveTracer();if(!q?.projectId||!q._client)return;if(!q.supportsLiveInstrumentation)return;let $=V._getCurrentTraceAndSpanId();if(!$)return;let[D]=$,P=Array.isArray(R)?R:[R];q._client.postV1projectsTracesByTraceIdTags(q.projectId,D,{tags:P}).catch((U)=>{F.error(`tag failed: ${String(U)}`)})})}static asyncEvaluate(R,f){S("BaseTracer.asyncEvaluate",()=>{let{judge:q,example:$}=R,P=V._getProxyProvider().getActiveTracer();if(!P?.projectId)return;if(!P.supportsLiveInstrumentation)return;let U=V._resolveSpan(f);if(!U?.isRecording())return;let O=P.getSpanProcessor(),E=U.spanContext(),Q=O.stateIncr(E,"pending_evals_count"),Y={project_id:P.projectId,eval_name:`async_evaluate_${q}_${Q}`,judges:[{name:q}],examples:[{...$,example_id:crypto.randomUUID(),created_at:new Date().toISOString(),trace_id:E.traceId,span_id:E.spanId}],is_offline:!1,is_behavior:!1},W=O.stateAppend(E,"pending_evals",Y);U.setAttribute("judgment.pending_trace_eval",JSON.stringify(W))})}}function wf(R,f){try{let q=Df(R).map((D)=>D.replace(/^\.\.\./,"").split("=")[0].trim()).filter((D)=>D.length>0),$={};return q.forEach((D,P)=>{if(P<f.length)$[D]=f[P]}),$}catch{return{}}}var hf="judgeval";var K=H(()=>{PR();S1();Pf();k();T();y();wR();w1();UR()});import{ROOT_CONTEXT as mq,context as o}from"@opentelemetry/api";import{AsyncLocalStorage as bf}from"async_hooks";function b1(){return Mf.getStore()===!0}function Tf(R){if(_1=R,_f)return;let f=o;f.active=()=>{if(!b1())return kq();let q=tR.getStore();if(q)return q;return _1?_1():mq},f.with=(q,$,D,...P)=>{if(!b1())return Lq(q,$,D,...P);return tR.run(q,()=>$.apply(D,P))},f.bind=(q,$)=>{if(!b1())return Nq(q,$);if(typeof $!=="function")return $;let D=$;return(...P)=>tR.run(q,()=>D(...P))},_f=!0}function M1(R,f){return Mf.run(!0,()=>tR.run(R,f))}var _f=!1,_1=null,Mf,tR,kq,Lq,Nq;var zf=H(()=>{Mf=new bf,tR=new bf,kq=o.active.bind(o),Lq=o.with.bind(o),Nq=o.bind.bind(o)});import{ROOT_CONTEXT as yq,SpanStatusCode as jf,trace as OR}from"@opentelemetry/api";import{registerInstrumentations as Aq}from"@opentelemetry/instrumentation";import{AsyncLocalStorage as Bq}from"async_hooks";class xf{_provider;constructor(R){this._provider=R}startSpan(R,f,q){let $=q??this._provider.getCurrentContext();return this._provider._getDelegateTracer().startSpan(R,f,$)}startActiveSpan(R,...f){let q={},$=this._provider.getCurrentContext(),D;if(f.length===1)D=f[0];else if(f.length===2)q=f[0],D=f[1];else q=f[0],$=f[1],D=f[2];let P=this.startSpan(R,q,$);return this._provider.useSpan(P,!1,!1,!1,()=>D(P))}}class j{static _instance=null;_activeTracer=null;_instrumentations=[];_proxyTracer;_tracers=new Set;constructor(){this._proxyTracer=new xf(this),Vf(this),Tf(()=>this.getCurrentContext())}static getInstance(){return j._instance??=new j,j._instance}static installAsGlobalTracerProvider(){let R=j.getInstance();return OR.setGlobalTracerProvider(R)}register(R){this._tracers.add(R)}deregister(R){this._tracers.delete(R)}setActive(R){let f=this.getCurrentSpan();if(f?.isRecording()){if(OR.getSpan(this.getCurrentContext())===f)return F.error("Cannot set_active() while a root span is active. Keeping existing tracer provider."),!1}return this.register(R),this._activeTracer=R,!0}getActiveTracer(){return this._activeTracer}getCurrentContext(){return cR.getStore()??yq}setSpan(R,f){return OR.setSpan(R,f)}wrapSpanContext(R){return OR.wrapSpanContext(R)}getCurrentSpan(){let R=this.getCurrentContext();return OR.getSpan(R)}hasActiveRootSpan(){if(!this.getCurrentSpan()?.isRecording())return!1;return!0}_getDelegateTracer(){let R=this._activeTracer;if(!R)return F.debug("No active tracer, returning NoOpTracer"),nR;return R._tracerProvider.getTracer(vq)}getTracer(R,f,q){return this._proxyTracer}addInstrumentation(R){try{Aq({tracerProvider:this,instrumentations:[R]}),this._instrumentations.push(R)}catch(f){F.error(`Failed to add instrumentation: ${String(f)}`)}}useSpan(R,f,q,$,D){let P=this.getCurrentContext(),U=OR.setSpan(P,R);return cR.run(U,()=>M1(U,()=>{try{let O=D();if(O instanceof Promise)return O.catch((E)=>{if(R.isRecording()){if(q)R.recordException(E);if($){let Q=E;R.setStatus({code:jf.ERROR,message:`${Q.name}: ${Q.message}`})}}throw E}).finally(()=>{if(f)R.end()});if(f)R.end();return O}catch(O){if(R.isRecording()){if(q)R.recordException(O);if($){let E=O;R.setStatus({code:jf.ERROR,message:`${E.name}: ${E.message}`})}}if(f)R.end();throw O}}))}attachContext(R){cR.enterWith(R)}withContext(R,f){return cR.run(R,()=>M1(R,f))}async forceFlush(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.forceFlush()));for(let f of R)if(f.status==="rejected")F.error(`forceFlush failed: ${String(f.reason)}`)}async shutdown(){let R=await Promise.allSettled(Array.from(this._tracers).map((f)=>f._tracerProvider.shutdown()));for(let f of R)if(f.status==="rejected")F.error(`shutdown failed: ${String(f.reason)}`);this._activeTracer=null,this._tracers.clear()}}var vq="judgeval",cR;var _R=H(()=>{T();zf();C1();UR();cR=new Bq});import{OTLPTraceExporter as Iq}from"@opentelemetry/exporter-trace-otlp-http";class v{_delegate;constructor(R,f,q,$){if(!R){this._delegate=null;return}this._delegate=new Iq({url:R,headers:{Authorization:`Bearer ${f}`,"X-Organization-Id":q,"X-Project-Id":$}})}export(R,f){F.info(`Exported ${R.length} spans`),this._delegate?.export(R,f)}shutdown(){return this._delegate?.shutdown()??Promise.resolve()}forceFlush(){return this._delegate?.forceFlush()??Promise.resolve()}}var bR=H(()=>{T()});import{ExportResultCode as Kq}from"@opentelemetry/core";var t;var rR=H(()=>{bR();t=class t extends v{constructor(){super("","","","")}export(R,f){f({code:Kq.SUCCESS})}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}}});class MR{_keyPredicate;constructor(R=T1){this._keyPredicate=R}onStart(R,f){let q=u(f)?.getAllEntries()??[];for(let[$,D]of q)if(this._keyPredicate($))R.setAttribute($,D.value)}onEnd(R){}forceFlush(){return Promise.resolve()}shutdown(){return Promise.resolve()}}var T1=()=>!0;var z1=H(()=>{wR()});import{BatchSpanProcessor as uq}from"@opentelemetry/sdk-trace-base";function VR(R){return`${R.traceId}:${R.spanId}`}function dq(R){return R[0]===0&&R[1]===0}var d;var TR=H(()=>{PR();k();UR();z1();d=class d extends uq{tracer;_state=new Map;_spanFinalizers;_baggageProcessor;constructor(R,f,q){super(f,q);this.tracer=R,this._spanFinalizers=new FinalizationRegistry(($)=>{this._cleanupSpanState($)}),this._baggageProcessor=new MR}_cleanupSpanState(R){this._state.delete(R)}_registerSpan(R){let f=R.spanContext();if(!f.traceId||!f.spanId)return;let q=VR(f);this._spanFinalizers.register(R,q)}stateSet(R,f,q){let $=VR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);D.set(f,q)}stateGet(R,f,q){let $=VR(R),D=this._state.get($);if(!D?.has(f))return q;return D.get(f)}stateIncr(R,f){let q=VR(R),$=this._state.get(q);if(!$)$=new Map,this._state.set(q,$);let D=$.get(f),P=typeof D==="number"?D:0;return $.set(f,P+1),P}stateAppend(R,f,q){let $=VR(R),D=this._state.get($);if(!D)D=new Map,this._state.set($,D);let P=D.get(f),U=Array.isArray(P)?[...P,q]:[q];return D.set(f,U),U}_emitSpan(R,f=!1){let q=R.spanContext();if(!q.traceId)return;let $=this.stateIncr(q,"judgment.update_id"),D={...R.attributes,"judgment.update_id":$};if(f)delete D["judgment.pending_trace_eval"];let P=Object.create(R);Object.defineProperty(P,"attributes",{value:D,writable:!1});let U=dq(R.endTime)?R.startTime:R.endTime;Object.defineProperty(P,"endTime",{value:U,writable:!1}),super.onEnd(P)}emitPartial(){S("JudgmentSpanProcessor.emitPartial",()=>{let R=B().getCurrentSpan();if(!R?.isRecording())return;let f=R.spanContext();if(!f.traceId)return;if(this.stateGet(f,"disable_partial_emit",!1))return;this._emitSpan(R,!0)})}onStart(R,f){S("JudgmentSpanProcessor.onStart",()=>{this._baggageProcessor.onStart(R,f),this._registerSpan(R)})}onEnd(R){S("JudgmentSpanProcessor.onEnd",()=>{let f=R.spanContext();if(!f.traceId){super.onEnd(R);return}let q=VR(f);try{if(!this.stateGet(f,"cancelled",!1))this._emitSpan(R)}finally{this._cleanupSpanState(q)}})}}});var zR;var j1=H(()=>{rR();TR();zR=class zR extends d{constructor(){super(null,new t)}onStart(R,f){}onEnd(R){}shutdown(){return Promise.resolve()}forceFlush(){return Promise.resolve()}emitPartial(){}stateSet(R,f,q){}stateGet(R,f,q){return q}stateIncr(R,f){return 0}stateAppend(R,f,q){return[q]}}});import{defaultResource as pq,resourceFromAttributes as lq}from"@opentelemetry/resources";import{NodeTracerProvider as mf}from"@opentelemetry/sdk-trace-node";var I;var eR=H(()=>{QR();yR();T();BR();y();V1();K();_R();bR();rR();TR();j1();I=class I extends V{_spanExporter=null;_spanProcessor=null;constructor(R,f,q,$,D,P,U,O,E,Q){super(R,f,q,$,D,P,U,O,E,Q)}static async init(R={}){let f=R.apiKey??a,q=R.organizationId??RR,$=R.apiUrl??fR,D=R.projectName??null,P=R.serializer??w,U=!0;if(!D)F.warning("project_name not provided. Tracer will not export spans."),U=!1;if(!f)F.warning("api_key not provided. Tracer will not export spans."),U=!1;if(!q)F.warning("organization_id not provided. Tracer will not export spans."),U=!1;if(!$)F.warning("api_url not provided. Tracer will not export spans."),U=!1;let O=null,E=null;if(U&&D&&f&&q&&$){if(O=new s($,f,q),E=await qR(O,D).catch(()=>null),!E)F.warning(`Project '${D}' not found. Tracer will not export spans.`),U=!1}let Q={"service.name":D??"unknown","telemetry.sdk.name":"judgeval","telemetry.sdk.version":lR};if(R.environment)Q["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(Q,R.resourceAttributes);let Y=pq().merge(lq(Q)),W=new mf({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits}),C=new I(D,E,f,q,$,R.environment??null,P,W,O,U);if(U){let h=new mf({resource:Y,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[C.getSpanProcessor(),...R.spanProcessors??[]]});C._tracerProvider=h}if(j.getInstance().register(C),R.setActive??!0)C.setActive();return C}getSpanExporter(){if(this._spanExporter)return this._spanExporter;if(!this._enableMonitoring||!this.projectId||!this.apiKey||!this.organizationId||!this.apiUrl)this._spanExporter=new t;else{let R=this.apiUrl.endsWith("/")?this.apiUrl+"otel/v1/traces":this.apiUrl+"/otel/v1/traces";this._spanExporter=new v(R,this.apiKey,this.organizationId,this.projectId)}return this._spanExporter}getSpanProcessor(){if(this._spanProcessor)return this._spanProcessor;if(!this._enableMonitoring)this._spanProcessor=new zR;else this._spanProcessor=new d(this,this.getSpanExporter());return this._spanProcessor}}});var jR;var x1=H(()=>{SR();TR();jR=class jR extends d{_dataset;_exampleFields;_seenTraceIds=new Set;constructor(R,f,q){super(R,f);this._dataset=q.dataset,this._exampleFields={...q.exampleFields??{}}}_maybeCreateExample(R){if(R.parentSpanContext)return;let f=R.spanContext();if(!f?.traceId)return;if(this._seenTraceIds.has(f.traceId))return;this._seenTraceIds.add(f.traceId);let q=m.create({...this._exampleFields,offline_trace_id:f.traceId});this._dataset.push(q)}onEnd(R){try{this._maybeCreateExample(R)}finally{super.onEnd(R)}}}});var Nf={};$1(Nf,{OfflineTracer:()=>c});import{defaultResource as sq,resourceFromAttributes as nq}from"@opentelemetry/resources";import{NodeTracerProvider as kf}from"@opentelemetry/sdk-trace-node";var Lf="otel/v1/offline-traces",c;var aR=H(()=>{QR();yR();BR();y();V1();_R();eR();bR();x1();c=class c extends I{supportsLiveInstrumentation=!1;_offlineApiUrl;_offlineApiKey;_offlineOrganizationId;_offlineProjectId;_dataset;_exampleFields;_offlineSpanExporter=null;_offlineSpanProcessor=null;constructor(R){super(R.projectName,R.projectId,R.apiKey,R.organizationId,R.apiUrl,R.environment,R.serializer,R.tracerProvider,R.client,!0);this._offlineApiUrl=R.apiUrl,this._offlineApiKey=R.apiKey,this._offlineOrganizationId=R.organizationId,this._offlineProjectId=R.projectId,this._dataset=R.dataset,this._exampleFields=R.exampleFields}static async create(R){let f=R.apiKey??a,q=R.organizationId??RR,$=R.apiUrl??fR,D=R.projectName,P=R.serializer??w;if(!D)throw Error("projectName is required for OfflineTracer");if(!f)throw Error("apiKey is required for OfflineTracer");if(!q)throw Error("organizationId is required for OfflineTracer");if(!$)throw Error("apiUrl is required for OfflineTracer");let U=new s($,f,q),O;try{O=await qR(U,D)}catch(G){throw Error(`Project '${D}' not found; cannot start OfflineTracer: ${String(G)}`)}let E={"service.name":D,"telemetry.sdk.name":"judgeval","telemetry.sdk.version":lR,"judgment.offline":"true"};if(R.environment)E["deployment.environment"]=R.environment;if(R.resourceAttributes)Object.assign(E,R.resourceAttributes);let Q=sq().merge(nq(E)),Y=new c({projectName:D,projectId:O,apiKey:f,organizationId:q,apiUrl:$,environment:R.environment??null,serializer:P,tracerProvider:new kf({resource:Q}),client:U,dataset:R.dataset,exampleFields:{...R.exampleFields??{}}}),W=new kf({resource:Q,sampler:R.sampler,spanLimits:R.spanLimits,spanProcessors:[Y.getSpanProcessor(),...R.spanProcessors??[]]});if(Y._tracerProvider=W,j.getInstance().register(Y),R.setActive??!0)Y.setActive();return Y}getSpanExporter(){if(this._offlineSpanExporter)return this._offlineSpanExporter;let R=this._offlineApiUrl.endsWith("/")?this._offlineApiUrl+Lf:this._offlineApiUrl+"/"+Lf;return this._offlineSpanExporter=new v(R,this._offlineApiKey,this._offlineOrganizationId,this._offlineProjectId),this._offlineSpanExporter}getSpanProcessor(){if(this._offlineSpanProcessor)return this._offlineSpanProcessor;return this._offlineSpanProcessor=new jR(this,this.getSpanExporter(),{dataset:this._dataset,exampleFields:this._exampleFields}),this._offlineSpanProcessor}}});QR();yR();BR();T();class YR{}T();var U1=FR(WR(),1);T();var J=FR(WR(),1),ef=2000;class $R{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async _poll(R,f,q,$){let D=Date.now();while(!0){let P=(Date.now()-D)/1000;if(P>$)throw Error(`Evaluation timed out after ${$}s`);let U=await this._client.getV1projectsExperimentsByRunId(R,f),O=U.results??[];if(O.length>=q){let Q=U.ui_results_url??"Failed to get UI results URL";return console.log(`${J.default.green("✓")} Evals completed and saved in ${J.default.bold(`${P.toFixed(1)}s`)}`),{results:O,url:Q}}await new Promise((Q)=>setTimeout(Q,ef))}}_displayResults(R,f,q,$){let D=[],P=0,U=0;console.log();for(let O=0;O<f.length;O++){let E=f[O],Q=E.scorers.every((Y)=>Boolean(Y.success));if(Q)P++,console.log(`${J.default.green("✓")} Example ${O+1}: ${J.default.green("PASSED")}`);else U++,console.log(`${J.default.red("✗")} Example ${O+1}: ${J.default.red("FAILED")}`);for(let Y of E.scorers){let W=Y.score!==null?Y.score.toFixed(3):"N/A",C=Y.success?J.default.green(W):J.default.red(W);console.log(` ${J.default.dim(`${Y.name}:`)} ${C} ${J.default.dim(`(threshold: ${Y.threshold})`)}`)}D.push({success:Q,scorers:E.scorers,example:R[O]})}if(console.log(),P===D.length)console.log(`${J.default.bold(J.default.green("✓ All tests passed!"))} (${P}/${D.length})`);else console.log(`${J.default.bold(J.default.yellow("⚠ Results:"))} ${J.default.green(`${P} passed`)} | ${J.default.red(`${U} failed`)}`);if(console.log(`${J.default.dim("View full details:")} ${J.default.underline(q)}`),console.log(),$&&D.some((O)=>!O.success)){let O=[`Evaluation failed: ${U}/${D.length} examples failed`];for(let E=0;E<D.length;E++)if(!D[E].success){O.push(` Example ${E+1}:`);for(let Q of D[E].scorers)if(!Q.success){if(O.push(` ${Q.name}: ${Q.score!==null?Q.score.toFixed(3):"N/A"} (threshold: ${Q.threshold})`),Q.reason)O.push(` ${Q.reason}`)}}throw Error(O.join(`
4
+ `))}return D}async run(R,f,q,$=!1,D=300){if(!this._projectId)return F.error("Project ID is not resolved. Evaluation requires a valid project."),[];let P=this._projectId,U=crypto.randomUUID(),O=new Date().toISOString();console.log(),console.log(J.default.bold(J.default.cyan("Starting Evaluation"))),console.log(`${J.default.dim("Run:")} ${q}`),console.log(`${J.default.dim("Project:")} ${this._projectName}`),console.log(`${J.default.dim("Examples:")} ${R.length} | ${J.default.dim("Scorers:")} ${f.length}`),console.log();let E=this._buildPayload(U,P,q,O,R,f),Q=await this._submit(P,U,R,f,E),{results:Y,url:W}=await this._poll(P,U,Q,D);return this._displayResults(R,Y,W,$)}}class KR extends $R{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:[],custom_scorers:[]}}async _submit(R,f,q,$,D){let P=Date.now(),U=q.flatMap((W,C)=>$.map((G)=>G.score(W).then((h)=>({exampleIdx:C,scorer:G,result:h,error:null})).catch((h)=>({exampleIdx:C,scorer:G,result:null,error:String(h)})))),O=await Promise.all(U),E=((Date.now()-P)/1000).toFixed(1);console.log(`${U1.default.green("✓")} Scoring completed in ${U1.default.bold(`${E}s`)}`);let Q=new Map;for(let W of O){let C=Q.get(W.exampleIdx);if(!C)C=[],Q.set(W.exampleIdx,C);C.push(W)}let Y=q.map((W,C)=>{return{scorers_data:(Q.get(C)??[]).map((h)=>{if(h.error!==null)return{scorer_name:h.scorer.constructor.name,value:0,reason:"",error:h.error};let X=h.result;return{scorer_name:h.scorer.constructor.name,value:X.value,reason:X.reason,...X.citations&&{citations:X.citations.map((Z)=>({span_id:Z.spanId,span_attribute:Z.spanAttribute}))}}}),data_object:W.toJSON()}});return await this._client.postV1projectsEvalResultsExamples(R,{results:Y,run:D}),q.length}}var u1=FR(WR(),1);class uR extends $R{_buildPayload(R,f,q,$,D,P){return{id:R,project_id:f,eval_name:q,created_at:$,examples:D.map((U)=>U.toJSON()),judgment_scorers:P.map((U)=>({name:U})),custom_scorers:[]}}async _submit(R,f,q,$,D){return await this._client.postV1projectsEvalQueueExamples(R,D),console.log(`${u1.default.green("✓")} Evaluation submitted`),q.length}}class HR{_local;_hosted;constructor(R,f,q){this._local=new KR(R,f,q),this._hosted=new uR(R,f,q)}run(R){let{examples:f,scorers:q,evalRunName:$,assertTest:D=!1,timeoutSeconds:P=300}=R,U=q.filter((E)=>typeof E==="string"),O=q.filter((E)=>E instanceof YR);if(O.length>0&&U.length>0)return F.error("Running both local and hosted scorers is not supported. Please run your evaluation with either local or hosted scorers, but not both."),Promise.resolve([]);if(O.length===0&&U.length===0)return F.error("No valid local or hosted scorers provided. Please provide at least one local or hosted scorer."),Promise.resolve([]);if(O.length>0)return this._local.run(f,O,$,D,P);return this._hosted.run(f,U,$,D,P)}}class dR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}create(){return new HR(this._client,this._projectId,this._projectName)}}SR();T();SR();class DR{name;projectId;projectName;datasetKind;examples;_client;constructor(R){this.name=R.name,this.projectId=R.projectId,this.projectName=R.projectName,this.datasetKind=R.datasetKind??"example",this.examples=R.examples??[],this._client=R.client??null}async addExamples(R,f=100){if(!this._client)return;for(let q=0;q<R.length;q+=f){let $=R.slice(q,q+f);await this._client.postV1projectsDatasetsByDatasetIdentifierExamples(this.projectId,this.name,{examples:$.map((D)=>D.toJSON())})}}async addFromJson(R,f=100){let{readFile:q}=await import("fs/promises"),$=await q(R,"utf-8"),P=JSON.parse($).map((U)=>{if(typeof U!=="object"||U===null)throw Error("Each item in the JSON array must be an object");return m.create(U)});await this.addExamples(P,f)}get length(){return this.examples.length}[Symbol.iterator](){return this.examples[Symbol.iterator]()}toString(){return`Dataset(name=${this.name}, examples=${this.examples.length})`}}var af="trace";function E1(R){if(typeof R!=="object"||R===null||Array.isArray(R))throw Error("Dataset schema must be a JSON object.");let f=R;if(f.type!=="object")throw Error('Dataset schema must declare top-level type "object".');let q=f.properties;if(typeof q!=="object"||q===null||Array.isArray(q))throw Error("Dataset schema must declare a 'properties' object.");let $=Object.entries(q).filter(([,D])=>typeof D==="object"&&D!==null&&D.type===af).map(([D])=>D);if($.length>1)throw Error(`A dataset may declare at most one trace column; found ${$.length}: ${$.join(", ")}.`)}function Rq(R){if(typeof R==="boolean")return"boolean";if(typeof R==="number")return Number.isInteger(R)?"integer":"number";if(typeof R==="string")return"string";if(Array.isArray(R))return"array";if(typeof R==="object"&&R!==null)return"object";return"string"}function O1(R){if(R.length===0)throw Error("Cannot infer a dataset schema without examples. Pass an explicit `schema` to client.datasets.create().");let f={},q=null;for(let $ of R){let D=new Set;for(let[P,U]of Object.entries($.properties)){if(U===null||U===void 0)continue;if(D.add(P),!(P in f))f[P]={type:Rq(U)}}if(q===null)q=D;else if(q.size!==D.size||![...q].every((P)=>D.has(P)))throw Error("All examples must share the same set of fields (dataset schemas require all declared properties). Pass an explicit `schema` or make all examples homogeneous.")}return{type:"object",properties:f}}class pR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async get(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsDatasetsByDatasetIdentifier(f,R),$=q.dataset_kind??"example",D=(q.examples??[]).map((P)=>m.from({...P.data??{},example_id:P.example_id,created_at:P.created_at??""}));return new DR({name:R,projectId:f,projectName:this._projectName,datasetKind:$,examples:D,client:this._client})}async create(R,f={}){let q=this._expectProjectId();if(!q)return null;let{examples:$=[],overwrite:D=!1,batchSize:P=100,schema:U}=f,O;if(U!==void 0)E1(U),O=U;else{if($.length===0)throw Error("Datasets require a JSON Schema. Pass `schema` to create(), or provide `examples` to infer one.");O=O1($)}await this._client.postV1projectsDatasets(q,{name:R,examples:[],dataset_kind:"example",overwrite:D,schema:O});let E=new DR({name:R,projectId:q,projectName:this._projectName,examples:$,client:this._client});if($.length>0)await E.addExamples($,P);return E}list(){let R=this._expectProjectId();if(!R)return Promise.resolve(null);return this._client.getV1projectsDatasets(R)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Dataset operations require a valid project."),null;return this._projectId}}T();class CR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async create(R){let f=this._expectProjectId();if(!f)return null;let q={name:R.name,prompt:R.prompt,model:R.model,score_type:R.scoreType};if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;return{judgeId:(await this._client.postV1projectsJudges(f,q)).judge_id,name:R.name,prompt:R.prompt,model:R.model,scoreType:R.scoreType,description:R.description??null,judgeDescription:R.judgeDescription??null,categories:R.categories??null,minScore:R.minScore??null,maxScore:R.maxScore??null,majorVersion:0,minorVersion:0}}async update(R){let f=this._expectProjectId();if(!f)return null;let q={};if(R.prompt!==void 0)q.prompt=R.prompt;if(R.model!==void 0)q.model=R.model;if(R.scoreType!==void 0)q.score_type=R.scoreType;if(R.description!==void 0)q.description=R.description;if(R.judgeDescription!==void 0)q.judge_description=R.judgeDescription;if(R.categories!==void 0)q.categories=R.categories;if(R.minScore!==void 0)q.min_score=R.minScore;if(R.maxScore!==void 0)q.max_score=R.maxScore;if(R.sourceMajorVersion!==void 0)q.source_major_version=R.sourceMajorVersion;if(R.sourceMinorVersion!==void 0)q.source_minor_version=R.sourceMinorVersion;if(R.targetMajorVersion!==void 0)q.target_major_version=R.targetMajorVersion;if(R.targetMinorVersion!==void 0)q.target_minor_version=R.targetMinorVersion;let $=await this._client.patchV1projectsJudgesByJudgeId(f,R.judgeId,q);return fq($)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Agent judge operations require a valid project."),null;return this._projectId}}function fq(R){let f=R.judge;return{judgeId:f.id,name:f.name,prompt:f.prompt??"",model:f.model??"",scoreType:f.score_type,description:f.description??null,judgeDescription:f.judge_description??null,categories:f.categories??null,minScore:f.min_score??null,maxScore:f.max_score??null,majorVersion:f.major_version??null,minorVersion:f.minor_version??null}}T();T();eR();aR();_R();function R1(R){let f=Array.isArray(R.judges)?R.judges.filter((q)=>typeof q==="object"&&q!==null):[];return{id:typeof R.id==="string"?R.id:"",name:typeof R.name==="string"?R.name:"",datasetId:typeof R.dataset_id==="string"?R.dataset_id:"",description:typeof R.description==="string"?R.description:null,createdAt:typeof R.created_at==="string"?R.created_at:null,judges:f}}function yf(R){let f=[];for(let q of R)for(let $ of q.scorers)if($.success!==null&&$.success!==void 0)f.push($.success);if(f.length===0)return null;return f.every((q)=>q)}var _=FR(WR(),1);function x(R){return typeof R==="object"&&R!==null&&!Array.isArray(R)?R:{}}function r(R){return Array.isArray(R)?R:[]}function b(R){return typeof R==="string"?R:""}function Af(R){return R?"Yes":"No"}function gq(R){let{score_type:f,bool_value:q,str_value:$,num_value:D}=R;if(f==="binary")return typeof q==="boolean"?Af(q):null;if(f==="categorical")return typeof $==="string"?$:null;if(f==="numeric")return typeof D==="number"?D:null;if(typeof q==="boolean")return Af(q);if(typeof $==="string")return $;if(typeof D==="number")return D;return null}function iq(R){if(typeof R==="object"&&R!==null){let f=R.text;return typeof f==="string"&&f?f:null}if(typeof R==="string"){try{let f=JSON.parse(R);if(typeof f==="object"&&f!==null){let q=f.text;if(typeof q==="string")return q||null}}catch{}return R||null}return null}function Bf(R){if(!R||R.length===0)return;let f=[];for(let q of R){if(!q.name&&!q.judgeId)throw Error("judgeVersions entries require a 'name' (or 'judgeId') key");let $={};if(q.judgeId!==void 0)$.judge_id=q.judgeId;if(q.name!==void 0)$.name=q.name;if(q.tag!==void 0)$.tag=q.tag;if(q.version!==void 0)$.version=q.version;if(q.majorVersion!==void 0)$.major_version=q.majorVersion;if(q.minorVersion!==void 0)$.minor_version=q.minorVersion;f.push($)}return f}function vf(R,f,q){let $=[];for(let D of R){let P=b(D.example_id),U=x(D.data),O=[];for(let E of r(D.scorers)){let Q=x(E),Y={judge_id:Q.judge_id,judge_major_version:Q.judge_major_version,judge_minor_version:Q.judge_minor_version},W=iq(Q.reason);if(W)Y.reason=W;O.push({name:b(Q.judge_name),value:gq(Q),scoreType:typeof Q.score_type==="string"?Q.score_type:null,error:typeof Q.error==="string"?Q.error:null,success:typeof Q.success==="boolean"?Q.success:null,metadata:Y})}if(q){let E=Boolean(q({...U},O));for(let Q of O)Q.success=E}$.push({exampleId:P,data:U,scorers:O,agentOfflineTraceId:f[P]})}return $}function If(R){if(R.status!=="completed")throw Error(`Test run ${R.testRunId} finished with status '${R.status}'`);let f=R.results.filter((q)=>q.scorers.some(($)=>$.success===!1)).map((q)=>q.exampleId);if(f.length>0||R.passed!==!0)throw Error(`Test run ${R.testRunId} failed its pass condition for ${f.length} example(s): ${JSON.stringify(f)}`)}function Kf(R,f,q){console.log(),console.log(_.default.bold(_.default.cyan("Starting Offline Test"))),console.log(`${_.default.dim("Config:")} ${R}`),console.log(`${_.default.dim("Project:")} ${f}`),console.log(`${_.default.dim("Examples:")} ${q}`),console.log()}function uf(R,f){if(console.log(),R.forEach((q,$)=>{if(q.scorers.some((P)=>P.success!==null&&P.success!==void 0)){let P=q.scorers.every((U)=>U.success===!0);console.log(P?`${_.default.green("✓")} Example ${$+1}: ${_.default.green("PASSED")}`:`${_.default.red("✗")} Example ${$+1}: ${_.default.red("FAILED")}`)}else console.log(`${_.default.cyan("•")} Example ${$+1}:`);for(let P of q.scorers){if(P.error){console.log(` ${_.default.dim(`${P.name}:`)} ${_.default.red(`error: ${P.error}`)}`);continue}let U=P.value===null?"N/A":String(P.value),O=P.success===!0?_.default.green(U):P.success===!1?_.default.red(U):U;console.log(` ${_.default.dim(`${P.name}:`)} ${O}`)}}),console.log(),f)console.log(`${_.default.dim("View full details:")} ${_.default.underline(f)}`),console.log()}var oq=new Set(["completed","error","cancelled"]),tq=100,cq=200,rq=2000,f1=1e4;class xR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async resolveDatasetVersion(R,f){let q=await this._client.getV1projectsDatasetsByDatasetIdentifierVersions(this._projectId,R.datasetId),$=r(q.versions).filter((D)=>typeof D==="object"&&D!==null).map((D)=>D);if(f===void 0){if($.length===0)throw Error(`Dataset of test config '${R.name}' has no versions`);return $.reduce((D,P)=>Number(P.version_number??0)>Number(D.version_number??0)?P:D)}if(typeof f==="number"){let D=$.find((P)=>Number(P.version_number??0)===f);if(D)return D}else{let D=$.find((P)=>P.version_id===f);if(D)return D}throw Error(`Dataset version ${JSON.stringify(f)} does not exist for the dataset of test config '${R.name}'`)}async fetchExamples(R,f){let q=[],$,D,P=0;for(;;){if(P>=f1)throw Error(`fetchExamples exceeded ${f1} pages for dataset of test config '${R.name}'`);let U=await this._client.getV1projectsDatasetsByDatasetIdentifierPage(this._projectId,R.datasetId,String(f),String(tq),$,D);P+=1;for(let Q of r(U.entries)){let Y=x(x(Q).example),W=Y.data;if(typeof W==="string")try{W=JSON.parse(W)}catch{W={}}q.push({exampleId:b(Y.example_id),data:x(W),offlineTraceId:typeof Y.offline_trace_id==="string"?Y.offline_trace_id:null,createdAt:typeof Y.created_at==="string"?Y.created_at:null})}let O=x(U.metadata),E=x(O.nextCursor);if(!O.hasMore||Object.keys(E).length===0)break;$=b(E.created_at),D=b(E.example_id)}return q}createTestRun(R,f={}){let q={test_config_id:R.id,source:f.source??"sdk"};if(f.name)q.name=f.name;if(typeof f.datasetVersion==="number")q.dataset_version_number=f.datasetVersion;else if(typeof f.datasetVersion==="string")q.dataset_version_id=f.datasetVersion;let $=Bf(f.judgeVersions);if($)q.judge_versions=$;let D=f.agentTraces??{};if(Object.keys(D).length>0)q.agent_traces=Object.entries(D).map(([P,U])=>({example_id:P,agent_offline_trace_id:U}));return this._client.postV1projectsTestRuns(this._projectId,q)}async runAgent(R,f){let q=[],$=j.getInstance().getActiveTracer();await c.create({projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl(),dataset:q,setActive:!0});let D={};try{let P=I.observe(R,{spanType:"agent"});for(let U of f){let O=q.length;try{await P(U.data)}catch(E){F.error(`Agent entrypoint raised for example ${U.exampleId}: ${String(E)}`)}for(let E of q.slice(O)){let Q=E.get("offline_trace_id");if(U.exampleId&&typeof Q==="string"){D[U.exampleId]=Q;break}}}}finally{if(await I.forceFlush(),$)$.setActive()}return D}async waitForCompletion(R,f){let q=Date.now();for(;;){if((Date.now()-q)/1000>f)throw Error(`Test run ${R} did not complete within ${f}s`);let $=await this._client.getV1projectsTestRunsByTestRunId(this._projectId,R),D=b(x($.test_run).status);if(oq.has(D))return D;await AR(rq)}}async fetchItems(R){let f=[],q="",$,D=0;for(;;){if(D>=f1)throw Error(`fetchItems exceeded ${f1} pages for test run ${R}`);let P=await this._client.getV1projectsTestRunsByTestRunIdItems(this._projectId,R,String(cq),$);D+=1;for(let O of r(P.results))f.push(x(O));if(!q)q=b(P.ui_results_url);let U=P.next_cursor;if(!P.has_more||typeof U!=="string"||!U)break;$=U}return{items:f,uiResultsUrl:q}}async reportSuccess(R,f,q,$){let D=new Map,P=new Map,U=(Y,W,C,G)=>`${Y}\x00${W}\x00${C}\x00${G}`,O=(Y,W)=>`${Y}\x00${W}`;for(let Y of r(f.evaluation_runs)){let W=x(Y),C=W.run_id;if(typeof C!=="string"||!C)continue;D.set(U(b(W.example_id),b(W.judge_id),Number(W.judge_major_version??0),Number(W.judge_minor_version??0)),C),P.set(O(b(W.example_id),b(W.judge_name)),C)}let E=new Map;for(let Y of $)E.set(Y.exampleId,Y);let Q=[];for(let Y of q){let W=b(Y.example_id),C=E.get(W),G=C?C.scorers.map((X)=>X.success??null):[],h=r(Y.scorers);for(let X=0;X<h.length;X+=1){let Z=x(h[X]),M=D.get(U(W,b(Z.judge_id),Number(Z.judge_major_version??0),Number(Z.judge_minor_version??0)))??P.get(O(W,b(Z.judge_name)));if(!M){F.warning(`No evaluation run ref for scorer ${b(Z.judge_name)} of example ${W}; skipping its success update`);continue}Q.push({evaluation_run_id:M,success:X<G.length?G[X]:null})}}if(Q.length===0)return;await this._client.patchV1projectsTestRunsByTestRunIdSuccess(this._projectId,R,{successes:Q})}async run(R,f={}){let{agentFunction:q,judgeVersions:$,datasetVersion:D,passConditionFn:P,assertTest:U=!1,timeoutSeconds:O=600,runName:E}=f;if(U&&!P)throw Error("assertTest=true requires a passConditionFn to decide whether each row passes.");let Q=await this.resolveDatasetVersion(R,D),Y=Number(Q.version_number??0),W=await this.fetchExamples(R,Y);Kf(R.name,this._projectName,W.length),F.info(`Dataset version ${Y}: ${W.length} example(s)`);let C=typeof D==="string"?D:Y,G={};if(q&&W.length>0)G=await this.runAgent(q,W);let h=await this.createTestRun(R,{datasetVersion:C,judgeVersions:$,agentTraces:G,name:E}),X=x(h.test_run),Z=b(X.id),M=b(h.ui_results_url);F.info(`Created test run ${Z} over ${W.length} examples`);let kR=await this.waitForCompletion(Z,O),{items:LR,uiResultsUrl:N}=await this.fetchItems(Z);if(N)M=N;let p=vf(LR,G,P);if(P)await this.reportSuccess(Z,h,LR,p);uf(p,M||void 0);let e={testRunId:Z,status:kR,uiResultsUrl:M||void 0,results:p,agentOfflineTraceIds:G,passed:yf(p)};if(U)If(e);return e}}var eq=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function m1(R){return eq.test(R)}class mR{_client;_projectId;_projectName;constructor(R,f,q){this._client=R,this._projectId=f,this._projectName=q}async createConfig(R,f,q,$){let D=this._expectProjectId();if(!D)return null;let P=[];for(let E of q)if(typeof E==="string")P.push(m1(E)?{judge_id:E}:{name:E});else if(E.judgeId)P.push({judge_id:E.judgeId});else if(E.name)P.push({name:E.name});else throw Error("judges entries must be judge names or objects with a 'judgeId' or 'name'");let U={name:R,judges:P};if($!==void 0)U.description=$;if(m1(f))U.dataset_id=f;else U.dataset_name=f;let O=await this._client.postV1projectsTestConfigs(D,U);return F.info(`Created test config ${R}`),R1(O.test_config??{})}async getConfig(R){let f=this._expectProjectId();if(!f)return null;if(m1(R)){let D=await this._client.getV1projectsTestConfigsByTestConfigId(f,R);return R1(D.test_config??{})}let $=(await this.listConfigs()??[]).find((D)=>D.name===R);if(!$)return F.error(`Test config '${R}' not found`),null;return $}async listConfigs(R){let f=this._expectProjectId();if(!f)return null;let q=await this._client.getV1projectsTestConfigs(f,R);return(Array.isArray(q.test_configs)?q.test_configs:[]).map((D)=>R1(D))}async deleteConfig(R){let f=this._expectProjectId();if(!f)return!1;return await this._client.deleteV1projectsTestConfigsByTestConfigId(f,R),F.info(`Deleted test config ${R}`),!0}async run(R,f={}){let q=this._expectProjectId();if(!q)return null;let $=typeof R==="string"?await this.getConfig(R):R;if(!$||!$.id)return null;return new xR(this._client,q,this._projectName).run($,f)}_expectProjectId(){if(!this._projectId)return F.error("Project ID is not resolved. Offline-test operations require a valid project."),null;return this._projectId}}class k1{_client;_projectName;_projectId;constructor(R,f,q){this._client=R,this._projectName=f,this._projectId=q}static async create(R){let f=R.apiKey??a,q=R.organizationId??RR,$=R.apiUrl??fR;if(!f)throw Error("API key is required");if(!q)throw Error("Organization ID is required");if(!$)throw Error("API URL is required");if(!R.projectName)throw Error("Project name is required");let D=new s($,f,q),P=null;try{P=await qR(D,R.projectName)}catch{F.warning(`Project '${R.projectName}' not found. Some operations requiring project_id will be skipped.`)}return new k1(D,R.projectName,P)}async offlineTracer(R){let{OfflineTracer:f}=await Promise.resolve().then(() => (aR(),Nf));return f.create({...R,projectName:this._projectName,apiKey:this._client.getApiKey(),organizationId:this._client.getOrganizationId(),apiUrl:this._client.getBaseUrl()})}get datasets(){return new pR(this._client,this._projectId,this._projectName)}get evaluation(){return new dR(this._client,this._projectId,this._projectName)}get agentJudges(){return new CR(this._client,this._projectId,this._projectName)}get offlineTests(){return new mR(this._client,this._projectId,this._projectName)}}K();bR();rR();_R();TR();j1();x1();z1();oR();wR();w1();eR();aR();S1();SR();export{W1 as wrapOpenAI,H1 as wrap,E1 as validateDatasetSchema,h1 as propagation,O1 as inferSchemaFromExamples,X1 as baggage,I as Tracer,c as OfflineTracer,mR as OfflineTestsFactory,xR as OfflineTestRunner,jR as OfflineJudgmentSpanProcessor,zR as NoOpSpanProcessor,t as NoOpSpanExporter,j as JudgmentTracerProvider,v as JudgmentSpanExporter,MR as JudgmentBaggageSpanProcessor,i as JudgmentBaggagePropagator,k1 as Judgeval,YR as Judge,m as Example,HR as Evaluation,DR as Dataset,V as BaseTracer,CR as AgentJudgeFactory,T1 as ALLOW_ALL_BAGGAGE_KEYS};
5
5
 
6
- //# debugId=EBF1264474225C7964756E2164756E21
6
+ //# debugId=5568DEC83850C92264756E2164756E21
7
7
  //# sourceMappingURL=index.mjs.map